2024-11-23 16:13:35,516 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-23 16:13:35,527 main DEBUG Took 0.009348 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-23 16:13:35,528 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-23 16:13:35,528 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-23 16:13:35,529 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-23 16:13:35,530 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-23 16:13:35,536 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-23 16:13:35,551 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-23 16:13:35,553 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-23 16:13:35,554 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-23 16:13:35,554 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-23 16:13:35,555 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-23 16:13:35,556 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-23 16:13:35,557 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-23 16:13:35,557 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-23 16:13:35,558 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-23 16:13:35,558 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-23 16:13:35,559 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-23 16:13:35,560 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-23 16:13:35,561 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-23 16:13:35,561 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-23 16:13:35,562 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-23 16:13:35,562 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-23 16:13:35,563 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-23 16:13:35,563 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-23 16:13:35,564 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-23 16:13:35,564 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-23 16:13:35,565 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-23 16:13:35,565 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-23 16:13:35,566 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-23 16:13:35,566 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-23 16:13:35,567 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-23 16:13:35,567 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-23 16:13:35,569 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-23 16:13:35,571 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-23 16:13:35,573 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-23 16:13:35,574 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-23 16:13:35,575 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-23 16:13:35,575 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-23 16:13:35,587 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-23 16:13:35,590 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-23 16:13:35,592 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-23 16:13:35,593 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-23 16:13:35,593 main DEBUG createAppenders(={Console}) 2024-11-23 16:13:35,594 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad initialized 2024-11-23 16:13:35,595 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-23 16:13:35,595 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad OK. 2024-11-23 16:13:35,596 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-23 16:13:35,596 main DEBUG OutputStream closed 2024-11-23 16:13:35,597 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-23 16:13:35,597 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-23 16:13:35,597 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@55634720 OK 2024-11-23 16:13:35,665 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-23 16:13:35,667 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-23 16:13:35,668 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-23 16:13:35,669 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-23 16:13:35,669 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-23 16:13:35,670 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-23 16:13:35,670 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-23 16:13:35,670 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-23 16:13:35,670 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-23 16:13:35,671 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-23 16:13:35,671 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-23 16:13:35,671 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-23 16:13:35,671 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-23 16:13:35,672 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-23 16:13:35,672 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-23 16:13:35,672 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-23 16:13:35,673 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-23 16:13:35,673 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-23 16:13:35,675 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-23 16:13:35,676 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@19835e64) with optional ClassLoader: null 2024-11-23 16:13:35,676 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-23 16:13:35,677 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@19835e64] started OK. 2024-11-23T16:13:35,914 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2703c80e-a8f3-3f6b-fe54-0323a389f54b 2024-11-23 16:13:35,917 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-23 16:13:35,918 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-23T16:13:35,926 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.TestAcidGuaranteesWithEagerPolicy timeout: 13 mins 2024-11-23T16:13:35,944 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-23T16:13:35,959 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2703c80e-a8f3-3f6b-fe54-0323a389f54b/cluster_d53936a7-f36f-e8d1-b808-86c188667d37, deleteOnExit=true 2024-11-23T16:13:35,959 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-23T16:13:35,960 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2703c80e-a8f3-3f6b-fe54-0323a389f54b/test.cache.data in system properties and HBase conf 2024-11-23T16:13:35,960 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2703c80e-a8f3-3f6b-fe54-0323a389f54b/hadoop.tmp.dir in system properties and HBase conf 2024-11-23T16:13:35,961 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2703c80e-a8f3-3f6b-fe54-0323a389f54b/hadoop.log.dir in system properties and HBase conf 2024-11-23T16:13:35,961 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2703c80e-a8f3-3f6b-fe54-0323a389f54b/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-23T16:13:35,962 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2703c80e-a8f3-3f6b-fe54-0323a389f54b/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-23T16:13:35,962 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-23T16:13:36,049 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-23T16:13:36,135 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-23T16:13:36,139 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2703c80e-a8f3-3f6b-fe54-0323a389f54b/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-23T16:13:36,139 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2703c80e-a8f3-3f6b-fe54-0323a389f54b/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-23T16:13:36,140 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2703c80e-a8f3-3f6b-fe54-0323a389f54b/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-23T16:13:36,140 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2703c80e-a8f3-3f6b-fe54-0323a389f54b/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-23T16:13:36,141 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2703c80e-a8f3-3f6b-fe54-0323a389f54b/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-23T16:13:36,141 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2703c80e-a8f3-3f6b-fe54-0323a389f54b/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-23T16:13:36,142 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2703c80e-a8f3-3f6b-fe54-0323a389f54b/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-23T16:13:36,142 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2703c80e-a8f3-3f6b-fe54-0323a389f54b/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-23T16:13:36,142 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2703c80e-a8f3-3f6b-fe54-0323a389f54b/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-23T16:13:36,143 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2703c80e-a8f3-3f6b-fe54-0323a389f54b/nfs.dump.dir in system properties and HBase conf 2024-11-23T16:13:36,143 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2703c80e-a8f3-3f6b-fe54-0323a389f54b/java.io.tmpdir in system properties and HBase conf 2024-11-23T16:13:36,144 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2703c80e-a8f3-3f6b-fe54-0323a389f54b/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-23T16:13:36,144 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2703c80e-a8f3-3f6b-fe54-0323a389f54b/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-23T16:13:36,144 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2703c80e-a8f3-3f6b-fe54-0323a389f54b/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-23T16:13:36,986 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-23T16:13:37,075 INFO [Time-limited test {}] log.Log(170): Logging initialized @2258ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-23T16:13:37,154 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-23T16:13:37,217 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-23T16:13:37,236 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-23T16:13:37,237 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-23T16:13:37,238 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-23T16:13:37,250 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-23T16:13:37,252 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6b23711f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2703c80e-a8f3-3f6b-fe54-0323a389f54b/hadoop.log.dir/,AVAILABLE} 2024-11-23T16:13:37,253 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@93c62df{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-23T16:13:37,439 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4ded99d3{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2703c80e-a8f3-3f6b-fe54-0323a389f54b/java.io.tmpdir/jetty-localhost-46403-hadoop-hdfs-3_4_1-tests_jar-_-any-8292938565450463608/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-23T16:13:37,446 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@557285fa{HTTP/1.1, (http/1.1)}{localhost:46403} 2024-11-23T16:13:37,446 INFO [Time-limited test {}] server.Server(415): Started @2631ms 2024-11-23T16:13:37,888 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-23T16:13:37,896 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-23T16:13:37,897 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-23T16:13:37,897 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-23T16:13:37,897 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-23T16:13:37,898 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4fd7fa7c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2703c80e-a8f3-3f6b-fe54-0323a389f54b/hadoop.log.dir/,AVAILABLE} 2024-11-23T16:13:37,899 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@435f9f96{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-23T16:13:38,021 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4269bd31{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2703c80e-a8f3-3f6b-fe54-0323a389f54b/java.io.tmpdir/jetty-localhost-39083-hadoop-hdfs-3_4_1-tests_jar-_-any-13127669727585617876/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-23T16:13:38,022 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@353d64f2{HTTP/1.1, (http/1.1)}{localhost:39083} 2024-11-23T16:13:38,023 INFO [Time-limited test {}] server.Server(415): Started @3207ms 2024-11-23T16:13:38,078 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-23T16:13:38,547 WARN [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2703c80e-a8f3-3f6b-fe54-0323a389f54b/cluster_d53936a7-f36f-e8d1-b808-86c188667d37/data/data1/current/BP-339017890-172.17.0.2-1732378416724/current, will proceed with Du for space computation calculation, 2024-11-23T16:13:38,547 WARN [Thread-73 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2703c80e-a8f3-3f6b-fe54-0323a389f54b/cluster_d53936a7-f36f-e8d1-b808-86c188667d37/data/data2/current/BP-339017890-172.17.0.2-1732378416724/current, will proceed with Du for space computation calculation, 2024-11-23T16:13:38,584 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-23T16:13:38,633 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2703c80e-a8f3-3f6b-fe54-0323a389f54b 2024-11-23T16:13:38,637 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x66185320c1cf740f with lease ID 0x57e9f1b00bc1f281: Processing first storage report for DS-d21e312f-c6a7-4ec2-b0a4-8b95af2982ef from datanode DatanodeRegistration(127.0.0.1:46331, datanodeUuid=b94a57d0-2203-4b65-9d9c-ccac2d973b37, infoPort=40181, infoSecurePort=0, ipcPort=44203, storageInfo=lv=-57;cid=testClusterID;nsid=948256933;c=1732378416724) 2024-11-23T16:13:38,638 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x66185320c1cf740f with lease ID 0x57e9f1b00bc1f281: from storage DS-d21e312f-c6a7-4ec2-b0a4-8b95af2982ef node DatanodeRegistration(127.0.0.1:46331, datanodeUuid=b94a57d0-2203-4b65-9d9c-ccac2d973b37, infoPort=40181, infoSecurePort=0, ipcPort=44203, storageInfo=lv=-57;cid=testClusterID;nsid=948256933;c=1732378416724), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-23T16:13:38,639 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x66185320c1cf740f with lease ID 0x57e9f1b00bc1f281: Processing first storage report for DS-c064e9d1-2aee-48dc-b238-07b149f7018e from datanode DatanodeRegistration(127.0.0.1:46331, datanodeUuid=b94a57d0-2203-4b65-9d9c-ccac2d973b37, infoPort=40181, infoSecurePort=0, ipcPort=44203, storageInfo=lv=-57;cid=testClusterID;nsid=948256933;c=1732378416724) 2024-11-23T16:13:38,639 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x66185320c1cf740f with lease ID 0x57e9f1b00bc1f281: from storage DS-c064e9d1-2aee-48dc-b238-07b149f7018e node DatanodeRegistration(127.0.0.1:46331, datanodeUuid=b94a57d0-2203-4b65-9d9c-ccac2d973b37, infoPort=40181, infoSecurePort=0, ipcPort=44203, storageInfo=lv=-57;cid=testClusterID;nsid=948256933;c=1732378416724), blocks: 0, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-23T16:13:38,711 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2703c80e-a8f3-3f6b-fe54-0323a389f54b/cluster_d53936a7-f36f-e8d1-b808-86c188667d37/zookeeper_0, clientPort=59338, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2703c80e-a8f3-3f6b-fe54-0323a389f54b/cluster_d53936a7-f36f-e8d1-b808-86c188667d37/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2703c80e-a8f3-3f6b-fe54-0323a389f54b/cluster_d53936a7-f36f-e8d1-b808-86c188667d37/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-23T16:13:38,727 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=59338 2024-11-23T16:13:38,743 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-23T16:13:38,747 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-23T16:13:39,049 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073741825_1001 (size=7) 2024-11-23T16:13:39,454 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca with version=8 2024-11-23T16:13:39,455 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/hbase-staging 2024-11-23T16:13:39,563 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-23T16:13:39,827 INFO [Time-limited test {}] client.ConnectionUtils(128): master/1a7c5470b472:0 server-side Connection retries=45 2024-11-23T16:13:39,839 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-23T16:13:39,840 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-23T16:13:39,845 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-23T16:13:39,845 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-23T16:13:39,845 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-23T16:13:40,026 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-23T16:13:40,111 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-23T16:13:40,120 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-23T16:13:40,124 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-23T16:13:40,151 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 3064 (auto-detected) 2024-11-23T16:13:40,152 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-23T16:13:40,170 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:45799 2024-11-23T16:13:40,191 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:45799 connecting to ZooKeeper ensemble=127.0.0.1:59338 2024-11-23T16:13:40,224 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:457990x0, quorum=127.0.0.1:59338, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-23T16:13:40,227 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:45799-0x101459ac8920000 connected 2024-11-23T16:13:40,258 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-23T16:13:40,260 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-23T16:13:40,269 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:45799-0x101459ac8920000, quorum=127.0.0.1:59338, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-23T16:13:40,273 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca, hbase.cluster.distributed=false 2024-11-23T16:13:40,299 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:45799-0x101459ac8920000, quorum=127.0.0.1:59338, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-23T16:13:40,303 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=45799 2024-11-23T16:13:40,304 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=45799 2024-11-23T16:13:40,304 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=45799 2024-11-23T16:13:40,305 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=45799 2024-11-23T16:13:40,305 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=45799 2024-11-23T16:13:40,413 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/1a7c5470b472:0 server-side Connection retries=45 2024-11-23T16:13:40,415 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-23T16:13:40,415 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-23T16:13:40,416 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-23T16:13:40,416 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-23T16:13:40,416 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-23T16:13:40,419 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-23T16:13:40,421 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-23T16:13:40,422 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:44715 2024-11-23T16:13:40,423 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:44715 connecting to ZooKeeper ensemble=127.0.0.1:59338 2024-11-23T16:13:40,425 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-23T16:13:40,429 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-23T16:13:40,437 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:447150x0, quorum=127.0.0.1:59338, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-23T16:13:40,438 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:44715-0x101459ac8920001 connected 2024-11-23T16:13:40,438 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:44715-0x101459ac8920001, quorum=127.0.0.1:59338, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-23T16:13:40,442 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-23T16:13:40,450 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-23T16:13:40,453 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:44715-0x101459ac8920001, quorum=127.0.0.1:59338, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-23T16:13:40,458 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:44715-0x101459ac8920001, quorum=127.0.0.1:59338, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-23T16:13:40,458 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=44715 2024-11-23T16:13:40,459 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=44715 2024-11-23T16:13:40,459 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=44715 2024-11-23T16:13:40,461 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=44715 2024-11-23T16:13:40,464 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=44715 2024-11-23T16:13:40,480 DEBUG [M:0;1a7c5470b472:45799 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;1a7c5470b472:45799 2024-11-23T16:13:40,481 INFO [master/1a7c5470b472:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/1a7c5470b472,45799,1732378419617 2024-11-23T16:13:40,488 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44715-0x101459ac8920001, quorum=127.0.0.1:59338, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-23T16:13:40,488 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45799-0x101459ac8920000, quorum=127.0.0.1:59338, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-23T16:13:40,490 DEBUG [master/1a7c5470b472:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:45799-0x101459ac8920000, quorum=127.0.0.1:59338, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/1a7c5470b472,45799,1732378419617 2024-11-23T16:13:40,513 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44715-0x101459ac8920001, quorum=127.0.0.1:59338, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-23T16:13:40,513 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45799-0x101459ac8920000, quorum=127.0.0.1:59338, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T16:13:40,514 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44715-0x101459ac8920001, quorum=127.0.0.1:59338, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T16:13:40,516 DEBUG [master/1a7c5470b472:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:45799-0x101459ac8920000, quorum=127.0.0.1:59338, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-23T16:13:40,517 INFO [master/1a7c5470b472:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/1a7c5470b472,45799,1732378419617 from backup master directory 2024-11-23T16:13:40,520 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45799-0x101459ac8920000, quorum=127.0.0.1:59338, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/1a7c5470b472,45799,1732378419617 2024-11-23T16:13:40,520 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44715-0x101459ac8920001, quorum=127.0.0.1:59338, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-23T16:13:40,521 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45799-0x101459ac8920000, quorum=127.0.0.1:59338, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-23T16:13:40,521 WARN [master/1a7c5470b472:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-23T16:13:40,521 INFO [master/1a7c5470b472:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=1a7c5470b472,45799,1732378419617 2024-11-23T16:13:40,524 INFO [master/1a7c5470b472:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-23T16:13:40,526 INFO [master/1a7c5470b472:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-23T16:13:40,591 DEBUG [master/1a7c5470b472:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/hbase.id] with ID: 34e0592b-548a-4d73-be6a-df1bafadfd92 2024-11-23T16:13:40,591 DEBUG [master/1a7c5470b472:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/.tmp/hbase.id 2024-11-23T16:13:40,604 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073741826_1002 (size=42) 2024-11-23T16:13:41,008 DEBUG [master/1a7c5470b472:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/.tmp/hbase.id]:[hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/hbase.id] 2024-11-23T16:13:41,068 INFO [master/1a7c5470b472:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-23T16:13:41,074 INFO [master/1a7c5470b472:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-23T16:13:41,103 INFO [master/1a7c5470b472:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 26ms. 2024-11-23T16:13:41,107 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44715-0x101459ac8920001, quorum=127.0.0.1:59338, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T16:13:41,107 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45799-0x101459ac8920000, quorum=127.0.0.1:59338, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T16:13:41,128 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073741827_1003 (size=196) 2024-11-23T16:13:41,151 INFO [master/1a7c5470b472:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-23T16:13:41,155 INFO [master/1a7c5470b472:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-23T16:13:41,176 DEBUG [master/1a7c5470b472:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:13:41,182 INFO [master/1a7c5470b472:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-23T16:13:41,232 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073741828_1004 (size=1189) 2024-11-23T16:13:41,652 INFO [master/1a7c5470b472:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/MasterData/data/master/store 2024-11-23T16:13:41,673 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073741829_1005 (size=34) 2024-11-23T16:13:42,080 INFO [master/1a7c5470b472:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-23T16:13:42,083 DEBUG [master/1a7c5470b472:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T16:13:42,084 DEBUG [master/1a7c5470b472:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-23T16:13:42,084 INFO [master/1a7c5470b472:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-23T16:13:42,084 DEBUG [master/1a7c5470b472:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-23T16:13:42,086 DEBUG [master/1a7c5470b472:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-23T16:13:42,086 DEBUG [master/1a7c5470b472:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-23T16:13:42,086 INFO [master/1a7c5470b472:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-23T16:13:42,087 DEBUG [master/1a7c5470b472:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732378422084Disabling compacts and flushes for region at 1732378422084Disabling writes for close at 1732378422086 (+2 ms)Writing region close event to WAL at 1732378422086Closed at 1732378422086 2024-11-23T16:13:42,089 WARN [master/1a7c5470b472:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/MasterData/data/master/store/.initializing 2024-11-23T16:13:42,090 DEBUG [master/1a7c5470b472:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/MasterData/WALs/1a7c5470b472,45799,1732378419617 2024-11-23T16:13:42,098 INFO [master/1a7c5470b472:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-23T16:13:42,112 INFO [master/1a7c5470b472:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=1a7c5470b472%2C45799%2C1732378419617, suffix=, logDir=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/MasterData/WALs/1a7c5470b472,45799,1732378419617, archiveDir=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/MasterData/oldWALs, maxLogs=10 2024-11-23T16:13:42,135 DEBUG [master/1a7c5470b472:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/MasterData/WALs/1a7c5470b472,45799,1732378419617/1a7c5470b472%2C45799%2C1732378419617.1732378422117, exclude list is [], retry=0 2024-11-23T16:13:42,152 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46331,DS-d21e312f-c6a7-4ec2-b0a4-8b95af2982ef,DISK] 2024-11-23T16:13:42,154 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-23T16:13:42,190 INFO [master/1a7c5470b472:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/MasterData/WALs/1a7c5470b472,45799,1732378419617/1a7c5470b472%2C45799%2C1732378419617.1732378422117 2024-11-23T16:13:42,191 DEBUG [master/1a7c5470b472:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40181:40181)] 2024-11-23T16:13:42,192 DEBUG [master/1a7c5470b472:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-23T16:13:42,192 DEBUG [master/1a7c5470b472:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T16:13:42,196 DEBUG [master/1a7c5470b472:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-23T16:13:42,197 DEBUG [master/1a7c5470b472:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-23T16:13:42,235 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-23T16:13:42,260 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-23T16:13:42,264 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:13:42,266 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-23T16:13:42,267 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-23T16:13:42,270 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-23T16:13:42,270 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:13:42,271 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T16:13:42,271 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-23T16:13:42,274 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-23T16:13:42,274 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:13:42,275 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T16:13:42,275 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-23T16:13:42,277 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-23T16:13:42,277 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:13:42,278 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T16:13:42,279 DEBUG [master/1a7c5470b472:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-23T16:13:42,282 DEBUG [master/1a7c5470b472:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-23T16:13:42,283 DEBUG [master/1a7c5470b472:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-23T16:13:42,288 DEBUG [master/1a7c5470b472:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-23T16:13:42,289 DEBUG [master/1a7c5470b472:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-23T16:13:42,292 DEBUG [master/1a7c5470b472:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-23T16:13:42,296 DEBUG [master/1a7c5470b472:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-23T16:13:42,300 DEBUG [master/1a7c5470b472:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-23T16:13:42,301 INFO [master/1a7c5470b472:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59853577, jitterRate=-0.10811220109462738}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-23T16:13:42,307 DEBUG [master/1a7c5470b472:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1732378422210Initializing all the Stores at 1732378422212 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732378422213 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732378422213Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732378422214 (+1 ms)Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732378422214Cleaning up temporary data from old regions at 1732378422289 (+75 ms)Region opened successfully at 1732378422306 (+17 ms) 2024-11-23T16:13:42,308 INFO [master/1a7c5470b472:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-23T16:13:42,343 DEBUG [master/1a7c5470b472:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e779548, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=1a7c5470b472/172.17.0.2:0 2024-11-23T16:13:42,377 INFO [master/1a7c5470b472:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-23T16:13:42,389 INFO [master/1a7c5470b472:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-23T16:13:42,389 INFO [master/1a7c5470b472:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-23T16:13:42,392 INFO [master/1a7c5470b472:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-23T16:13:42,393 INFO [master/1a7c5470b472:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-23T16:13:42,398 INFO [master/1a7c5470b472:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-23T16:13:42,399 INFO [master/1a7c5470b472:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-23T16:13:42,426 INFO [master/1a7c5470b472:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-23T16:13:42,434 DEBUG [master/1a7c5470b472:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45799-0x101459ac8920000, quorum=127.0.0.1:59338, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-23T16:13:42,437 DEBUG [master/1a7c5470b472:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-23T16:13:42,439 INFO [master/1a7c5470b472:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-23T16:13:42,441 DEBUG [master/1a7c5470b472:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45799-0x101459ac8920000, quorum=127.0.0.1:59338, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-23T16:13:42,444 DEBUG [master/1a7c5470b472:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-23T16:13:42,446 INFO [master/1a7c5470b472:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-23T16:13:42,450 DEBUG [master/1a7c5470b472:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45799-0x101459ac8920000, quorum=127.0.0.1:59338, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-23T16:13:42,452 DEBUG [master/1a7c5470b472:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-23T16:13:42,453 DEBUG [master/1a7c5470b472:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45799-0x101459ac8920000, quorum=127.0.0.1:59338, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-23T16:13:42,455 DEBUG [master/1a7c5470b472:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-23T16:13:42,473 DEBUG [master/1a7c5470b472:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45799-0x101459ac8920000, quorum=127.0.0.1:59338, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-23T16:13:42,475 DEBUG [master/1a7c5470b472:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-23T16:13:42,481 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44715-0x101459ac8920001, quorum=127.0.0.1:59338, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-23T16:13:42,481 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45799-0x101459ac8920000, quorum=127.0.0.1:59338, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-23T16:13:42,481 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44715-0x101459ac8920001, quorum=127.0.0.1:59338, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T16:13:42,481 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45799-0x101459ac8920000, quorum=127.0.0.1:59338, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T16:13:42,483 INFO [master/1a7c5470b472:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=1a7c5470b472,45799,1732378419617, sessionid=0x101459ac8920000, setting cluster-up flag (Was=false) 2024-11-23T16:13:42,499 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44715-0x101459ac8920001, quorum=127.0.0.1:59338, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T16:13:42,499 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45799-0x101459ac8920000, quorum=127.0.0.1:59338, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T16:13:42,506 DEBUG [master/1a7c5470b472:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-23T16:13:42,509 DEBUG [master/1a7c5470b472:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=1a7c5470b472,45799,1732378419617 2024-11-23T16:13:42,514 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44715-0x101459ac8920001, quorum=127.0.0.1:59338, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T16:13:42,514 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45799-0x101459ac8920000, quorum=127.0.0.1:59338, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T16:13:42,521 DEBUG [master/1a7c5470b472:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-23T16:13:42,522 DEBUG [master/1a7c5470b472:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=1a7c5470b472,45799,1732378419617 2024-11-23T16:13:42,528 INFO [master/1a7c5470b472:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-23T16:13:42,570 INFO [RS:0;1a7c5470b472:44715 {}] regionserver.HRegionServer(746): ClusterId : 34e0592b-548a-4d73-be6a-df1bafadfd92 2024-11-23T16:13:42,574 DEBUG [RS:0;1a7c5470b472:44715 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-23T16:13:42,579 DEBUG [RS:0;1a7c5470b472:44715 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-23T16:13:42,579 DEBUG [RS:0;1a7c5470b472:44715 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-23T16:13:42,584 DEBUG [RS:0;1a7c5470b472:44715 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-23T16:13:42,585 DEBUG [RS:0;1a7c5470b472:44715 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f36baba, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=1a7c5470b472/172.17.0.2:0 2024-11-23T16:13:42,599 DEBUG [RS:0;1a7c5470b472:44715 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;1a7c5470b472:44715 2024-11-23T16:13:42,603 INFO [RS:0;1a7c5470b472:44715 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-23T16:13:42,603 INFO [RS:0;1a7c5470b472:44715 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-23T16:13:42,603 DEBUG [RS:0;1a7c5470b472:44715 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-23T16:13:42,606 INFO [RS:0;1a7c5470b472:44715 {}] regionserver.HRegionServer(2659): reportForDuty to master=1a7c5470b472,45799,1732378419617 with port=44715, startcode=1732378420375 2024-11-23T16:13:42,608 DEBUG [master/1a7c5470b472:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-23T16:13:42,619 INFO [master/1a7c5470b472:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-23T16:13:42,620 DEBUG [RS:0;1a7c5470b472:44715 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-23T16:13:42,627 INFO [master/1a7c5470b472:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-23T16:13:42,635 DEBUG [master/1a7c5470b472:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 1a7c5470b472,45799,1732378419617 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-23T16:13:42,645 DEBUG [master/1a7c5470b472:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/1a7c5470b472:0, corePoolSize=5, maxPoolSize=5 2024-11-23T16:13:42,645 DEBUG [master/1a7c5470b472:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/1a7c5470b472:0, corePoolSize=5, maxPoolSize=5 2024-11-23T16:13:42,645 DEBUG [master/1a7c5470b472:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/1a7c5470b472:0, corePoolSize=5, maxPoolSize=5 2024-11-23T16:13:42,646 DEBUG [master/1a7c5470b472:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/1a7c5470b472:0, corePoolSize=5, maxPoolSize=5 2024-11-23T16:13:42,646 DEBUG [master/1a7c5470b472:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/1a7c5470b472:0, corePoolSize=10, maxPoolSize=10 2024-11-23T16:13:42,646 DEBUG [master/1a7c5470b472:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/1a7c5470b472:0, corePoolSize=1, maxPoolSize=1 2024-11-23T16:13:42,647 DEBUG [master/1a7c5470b472:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/1a7c5470b472:0, corePoolSize=2, maxPoolSize=2 2024-11-23T16:13:42,647 DEBUG [master/1a7c5470b472:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/1a7c5470b472:0, corePoolSize=1, maxPoolSize=1 2024-11-23T16:13:42,653 INFO [master/1a7c5470b472:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732378452653 2024-11-23T16:13:42,655 INFO [master/1a7c5470b472:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-23T16:13:42,655 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-23T16:13:42,655 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-23T16:13:42,656 INFO [master/1a7c5470b472:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-23T16:13:42,663 INFO [master/1a7c5470b472:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-23T16:13:42,664 INFO [master/1a7c5470b472:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-23T16:13:42,664 INFO [master/1a7c5470b472:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-23T16:13:42,664 INFO [master/1a7c5470b472:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-23T16:13:42,665 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44831, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-23T16:13:42,665 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:13:42,666 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-23T16:13:42,668 INFO [master/1a7c5470b472:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-23T16:13:42,675 INFO [master/1a7c5470b472:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-23T16:13:42,676 INFO [master/1a7c5470b472:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-23T16:13:42,677 INFO [master/1a7c5470b472:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-23T16:13:42,677 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45799 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:13:42,685 INFO [master/1a7c5470b472:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-23T16:13:42,685 INFO [master/1a7c5470b472:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-23T16:13:42,688 DEBUG [master/1a7c5470b472:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/1a7c5470b472:0:becomeActiveMaster-HFileCleaner.large.0-1732378422687,5,FailOnTimeoutGroup] 2024-11-23T16:13:42,690 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073741831_1007 (size=1321) 2024-11-23T16:13:42,692 DEBUG [master/1a7c5470b472:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/1a7c5470b472:0:becomeActiveMaster-HFileCleaner.small.0-1732378422688,5,FailOnTimeoutGroup] 2024-11-23T16:13:42,693 INFO [master/1a7c5470b472:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-23T16:13:42,693 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-23T16:13:42,693 INFO [master/1a7c5470b472:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-23T16:13:42,693 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca 2024-11-23T16:13:42,694 INFO [master/1a7c5470b472:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-23T16:13:42,695 INFO [master/1a7c5470b472:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-23T16:13:42,706 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073741832_1008 (size=32) 2024-11-23T16:13:42,710 DEBUG [RS:0;1a7c5470b472:44715 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-11-23T16:13:42,710 WARN [RS:0;1a7c5470b472:44715 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-11-23T16:13:42,811 INFO [RS:0;1a7c5470b472:44715 {}] regionserver.HRegionServer(2659): reportForDuty to master=1a7c5470b472,45799,1732378419617 with port=44715, startcode=1732378420375 2024-11-23T16:13:42,814 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45799 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 1a7c5470b472,44715,1732378420375 2024-11-23T16:13:42,816 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45799 {}] master.ServerManager(517): Registering regionserver=1a7c5470b472,44715,1732378420375 2024-11-23T16:13:42,825 DEBUG [RS:0;1a7c5470b472:44715 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca 2024-11-23T16:13:42,825 DEBUG [RS:0;1a7c5470b472:44715 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:39667 2024-11-23T16:13:42,825 DEBUG [RS:0;1a7c5470b472:44715 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-23T16:13:42,830 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45799-0x101459ac8920000, quorum=127.0.0.1:59338, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-23T16:13:42,830 DEBUG [RS:0;1a7c5470b472:44715 {}] zookeeper.ZKUtil(111): regionserver:44715-0x101459ac8920001, quorum=127.0.0.1:59338, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/1a7c5470b472,44715,1732378420375 2024-11-23T16:13:42,830 WARN [RS:0;1a7c5470b472:44715 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-23T16:13:42,831 INFO [RS:0;1a7c5470b472:44715 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-23T16:13:42,831 DEBUG [RS:0;1a7c5470b472:44715 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/WALs/1a7c5470b472,44715,1732378420375 2024-11-23T16:13:42,832 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [1a7c5470b472,44715,1732378420375] 2024-11-23T16:13:42,856 INFO [RS:0;1a7c5470b472:44715 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-23T16:13:42,870 INFO [RS:0;1a7c5470b472:44715 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-23T16:13:42,875 INFO [RS:0;1a7c5470b472:44715 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-23T16:13:42,875 INFO [RS:0;1a7c5470b472:44715 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-23T16:13:42,876 INFO [RS:0;1a7c5470b472:44715 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-23T16:13:42,882 INFO [RS:0;1a7c5470b472:44715 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-23T16:13:42,884 INFO [RS:0;1a7c5470b472:44715 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-23T16:13:42,884 DEBUG [RS:0;1a7c5470b472:44715 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/1a7c5470b472:0, corePoolSize=1, maxPoolSize=1 2024-11-23T16:13:42,884 DEBUG [RS:0;1a7c5470b472:44715 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/1a7c5470b472:0, corePoolSize=1, maxPoolSize=1 2024-11-23T16:13:42,884 DEBUG [RS:0;1a7c5470b472:44715 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/1a7c5470b472:0, corePoolSize=1, maxPoolSize=1 2024-11-23T16:13:42,885 DEBUG [RS:0;1a7c5470b472:44715 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/1a7c5470b472:0, corePoolSize=1, maxPoolSize=1 2024-11-23T16:13:42,885 DEBUG [RS:0;1a7c5470b472:44715 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/1a7c5470b472:0, corePoolSize=1, maxPoolSize=1 2024-11-23T16:13:42,885 DEBUG [RS:0;1a7c5470b472:44715 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/1a7c5470b472:0, corePoolSize=2, maxPoolSize=2 2024-11-23T16:13:42,885 DEBUG [RS:0;1a7c5470b472:44715 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/1a7c5470b472:0, corePoolSize=1, maxPoolSize=1 2024-11-23T16:13:42,885 DEBUG [RS:0;1a7c5470b472:44715 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/1a7c5470b472:0, corePoolSize=1, maxPoolSize=1 2024-11-23T16:13:42,885 DEBUG [RS:0;1a7c5470b472:44715 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/1a7c5470b472:0, corePoolSize=1, maxPoolSize=1 2024-11-23T16:13:42,885 DEBUG [RS:0;1a7c5470b472:44715 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/1a7c5470b472:0, corePoolSize=1, maxPoolSize=1 2024-11-23T16:13:42,886 DEBUG [RS:0;1a7c5470b472:44715 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/1a7c5470b472:0, corePoolSize=1, maxPoolSize=1 2024-11-23T16:13:42,886 DEBUG [RS:0;1a7c5470b472:44715 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/1a7c5470b472:0, corePoolSize=1, maxPoolSize=1 2024-11-23T16:13:42,886 DEBUG [RS:0;1a7c5470b472:44715 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/1a7c5470b472:0, corePoolSize=3, maxPoolSize=3 2024-11-23T16:13:42,886 DEBUG [RS:0;1a7c5470b472:44715 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0, corePoolSize=3, maxPoolSize=3 2024-11-23T16:13:42,887 INFO [RS:0;1a7c5470b472:44715 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-23T16:13:42,887 INFO [RS:0;1a7c5470b472:44715 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-23T16:13:42,887 INFO [RS:0;1a7c5470b472:44715 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-23T16:13:42,887 INFO [RS:0;1a7c5470b472:44715 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-23T16:13:42,888 INFO [RS:0;1a7c5470b472:44715 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-23T16:13:42,888 INFO [RS:0;1a7c5470b472:44715 {}] hbase.ChoreService(168): Chore ScheduledChore name=1a7c5470b472,44715,1732378420375-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-23T16:13:42,913 INFO [RS:0;1a7c5470b472:44715 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-23T16:13:42,915 INFO [RS:0;1a7c5470b472:44715 {}] hbase.ChoreService(168): Chore ScheduledChore name=1a7c5470b472,44715,1732378420375-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-23T16:13:42,916 INFO [RS:0;1a7c5470b472:44715 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-23T16:13:42,916 INFO [RS:0;1a7c5470b472:44715 {}] regionserver.Replication(171): 1a7c5470b472,44715,1732378420375 started 2024-11-23T16:13:42,936 INFO [RS:0;1a7c5470b472:44715 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-23T16:13:42,936 INFO [RS:0;1a7c5470b472:44715 {}] regionserver.HRegionServer(1482): Serving as 1a7c5470b472,44715,1732378420375, RpcServer on 1a7c5470b472/172.17.0.2:44715, sessionid=0x101459ac8920001 2024-11-23T16:13:42,937 DEBUG [RS:0;1a7c5470b472:44715 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-23T16:13:42,937 DEBUG [RS:0;1a7c5470b472:44715 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 1a7c5470b472,44715,1732378420375 2024-11-23T16:13:42,938 DEBUG [RS:0;1a7c5470b472:44715 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '1a7c5470b472,44715,1732378420375' 2024-11-23T16:13:42,938 DEBUG [RS:0;1a7c5470b472:44715 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-23T16:13:42,939 DEBUG [RS:0;1a7c5470b472:44715 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-23T16:13:42,940 DEBUG [RS:0;1a7c5470b472:44715 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-23T16:13:42,940 DEBUG [RS:0;1a7c5470b472:44715 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-23T16:13:42,940 DEBUG [RS:0;1a7c5470b472:44715 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 1a7c5470b472,44715,1732378420375 2024-11-23T16:13:42,940 DEBUG [RS:0;1a7c5470b472:44715 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '1a7c5470b472,44715,1732378420375' 2024-11-23T16:13:42,940 DEBUG [RS:0;1a7c5470b472:44715 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-23T16:13:42,941 DEBUG [RS:0;1a7c5470b472:44715 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-23T16:13:42,941 DEBUG [RS:0;1a7c5470b472:44715 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-23T16:13:42,941 INFO [RS:0;1a7c5470b472:44715 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-23T16:13:42,941 INFO [RS:0;1a7c5470b472:44715 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-23T16:13:43,047 INFO [RS:0;1a7c5470b472:44715 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-23T16:13:43,050 INFO [RS:0;1a7c5470b472:44715 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=1a7c5470b472%2C44715%2C1732378420375, suffix=, logDir=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/WALs/1a7c5470b472,44715,1732378420375, archiveDir=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/oldWALs, maxLogs=32 2024-11-23T16:13:43,068 DEBUG [RS:0;1a7c5470b472:44715 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/WALs/1a7c5470b472,44715,1732378420375/1a7c5470b472%2C44715%2C1732378420375.1732378423053, exclude list is [], retry=0 2024-11-23T16:13:43,073 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46331,DS-d21e312f-c6a7-4ec2-b0a4-8b95af2982ef,DISK] 2024-11-23T16:13:43,076 INFO [RS:0;1a7c5470b472:44715 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/WALs/1a7c5470b472,44715,1732378420375/1a7c5470b472%2C44715%2C1732378420375.1732378423053 2024-11-23T16:13:43,077 DEBUG [RS:0;1a7c5470b472:44715 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40181:40181)] 2024-11-23T16:13:43,109 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T16:13:43,111 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-23T16:13:43,114 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-23T16:13:43,114 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:13:43,115 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-23T16:13:43,115 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-23T16:13:43,117 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-23T16:13:43,118 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:13:43,118 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-23T16:13:43,119 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-23T16:13:43,121 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-23T16:13:43,121 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:13:43,122 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-23T16:13:43,122 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-23T16:13:43,124 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-23T16:13:43,124 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:13:43,125 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-23T16:13:43,125 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-23T16:13:43,126 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/hbase/meta/1588230740 2024-11-23T16:13:43,127 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/hbase/meta/1588230740 2024-11-23T16:13:43,129 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-23T16:13:43,130 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-23T16:13:43,130 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-23T16:13:43,133 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-23T16:13:43,137 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-23T16:13:43,138 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65094477, jitterRate=-0.0300167053937912}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-23T16:13:43,141 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1732378423109Initializing all the Stores at 1732378423111 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732378423111Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732378423111Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732378423111Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732378423111Cleaning up temporary data from old regions at 1732378423130 (+19 ms)Region opened successfully at 1732378423140 (+10 ms) 2024-11-23T16:13:43,141 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-23T16:13:43,141 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-23T16:13:43,141 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-23T16:13:43,141 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-23T16:13:43,142 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-23T16:13:43,144 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-23T16:13:43,144 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732378423141Disabling compacts and flushes for region at 1732378423141Disabling writes for close at 1732378423142 (+1 ms)Writing region close event to WAL at 1732378423143 (+1 ms)Closed at 1732378423143 2024-11-23T16:13:43,148 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-23T16:13:43,148 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-23T16:13:43,155 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-23T16:13:43,165 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-23T16:13:43,169 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-23T16:13:43,322 DEBUG [1a7c5470b472:45799 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-23T16:13:43,337 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=1a7c5470b472,44715,1732378420375 2024-11-23T16:13:43,345 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 1a7c5470b472,44715,1732378420375, state=OPENING 2024-11-23T16:13:43,352 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-23T16:13:43,354 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45799-0x101459ac8920000, quorum=127.0.0.1:59338, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T16:13:43,354 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44715-0x101459ac8920001, quorum=127.0.0.1:59338, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T16:13:43,355 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-23T16:13:43,355 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-23T16:13:43,356 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-23T16:13:43,358 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=1a7c5470b472,44715,1732378420375}] 2024-11-23T16:13:43,540 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-23T16:13:43,545 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35499, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-23T16:13:43,558 INFO [RS_OPEN_META-regionserver/1a7c5470b472:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-23T16:13:43,559 INFO [RS_OPEN_META-regionserver/1a7c5470b472:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-23T16:13:43,560 INFO [RS_OPEN_META-regionserver/1a7c5470b472:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-23T16:13:43,564 INFO [RS_OPEN_META-regionserver/1a7c5470b472:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=1a7c5470b472%2C44715%2C1732378420375.meta, suffix=.meta, logDir=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/WALs/1a7c5470b472,44715,1732378420375, archiveDir=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/oldWALs, maxLogs=32 2024-11-23T16:13:43,588 DEBUG [RS_OPEN_META-regionserver/1a7c5470b472:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/WALs/1a7c5470b472,44715,1732378420375/1a7c5470b472%2C44715%2C1732378420375.meta.1732378423567.meta, exclude list is [], retry=0 2024-11-23T16:13:43,593 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46331,DS-d21e312f-c6a7-4ec2-b0a4-8b95af2982ef,DISK] 2024-11-23T16:13:43,602 INFO [RS_OPEN_META-regionserver/1a7c5470b472:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/WALs/1a7c5470b472,44715,1732378420375/1a7c5470b472%2C44715%2C1732378420375.meta.1732378423567.meta 2024-11-23T16:13:43,603 DEBUG [RS_OPEN_META-regionserver/1a7c5470b472:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40181:40181)] 2024-11-23T16:13:43,603 DEBUG [RS_OPEN_META-regionserver/1a7c5470b472:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-23T16:13:43,605 DEBUG [RS_OPEN_META-regionserver/1a7c5470b472:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-23T16:13:43,609 DEBUG [RS_OPEN_META-regionserver/1a7c5470b472:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-23T16:13:43,615 INFO [RS_OPEN_META-regionserver/1a7c5470b472:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-23T16:13:43,621 DEBUG [RS_OPEN_META-regionserver/1a7c5470b472:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-23T16:13:43,622 DEBUG [RS_OPEN_META-regionserver/1a7c5470b472:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T16:13:43,622 DEBUG [RS_OPEN_META-regionserver/1a7c5470b472:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-23T16:13:43,622 DEBUG [RS_OPEN_META-regionserver/1a7c5470b472:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-23T16:13:43,626 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-23T16:13:43,628 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-23T16:13:43,628 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:13:43,630 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-23T16:13:43,630 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-23T16:13:43,632 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-23T16:13:43,632 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:13:43,633 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-23T16:13:43,633 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-23T16:13:43,635 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-23T16:13:43,635 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:13:43,636 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-23T16:13:43,636 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-23T16:13:43,639 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-23T16:13:43,639 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:13:43,640 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-23T16:13:43,640 DEBUG [RS_OPEN_META-regionserver/1a7c5470b472:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-23T16:13:43,642 DEBUG [RS_OPEN_META-regionserver/1a7c5470b472:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/hbase/meta/1588230740 2024-11-23T16:13:43,645 DEBUG [RS_OPEN_META-regionserver/1a7c5470b472:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/hbase/meta/1588230740 2024-11-23T16:13:43,649 DEBUG [RS_OPEN_META-regionserver/1a7c5470b472:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-23T16:13:43,649 DEBUG [RS_OPEN_META-regionserver/1a7c5470b472:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-23T16:13:43,650 DEBUG [RS_OPEN_META-regionserver/1a7c5470b472:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-23T16:13:43,653 DEBUG [RS_OPEN_META-regionserver/1a7c5470b472:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-23T16:13:43,655 INFO [RS_OPEN_META-regionserver/1a7c5470b472:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67026641, jitterRate=-0.0012252181768417358}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-23T16:13:43,656 DEBUG [RS_OPEN_META-regionserver/1a7c5470b472:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-23T16:13:43,658 DEBUG [RS_OPEN_META-regionserver/1a7c5470b472:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732378423623Writing region info on filesystem at 1732378423623Initializing all the Stores at 1732378423625 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732378423626 (+1 ms)Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732378423626Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732378423626Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732378423626Cleaning up temporary data from old regions at 1732378423649 (+23 ms)Running coprocessor post-open hooks at 1732378423656 (+7 ms)Region opened successfully at 1732378423658 (+2 ms) 2024-11-23T16:13:43,693 INFO [RS_OPEN_META-regionserver/1a7c5470b472:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732378423529 2024-11-23T16:13:43,708 DEBUG [RS_OPEN_META-regionserver/1a7c5470b472:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-23T16:13:43,709 INFO [RS_OPEN_META-regionserver/1a7c5470b472:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-23T16:13:43,712 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=1a7c5470b472,44715,1732378420375 2024-11-23T16:13:43,715 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 1a7c5470b472,44715,1732378420375, state=OPEN 2024-11-23T16:13:43,728 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44715-0x101459ac8920001, quorum=127.0.0.1:59338, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-23T16:13:43,728 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-23T16:13:43,728 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45799-0x101459ac8920000, quorum=127.0.0.1:59338, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-23T16:13:43,729 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-23T16:13:43,731 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:13:43,738 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-23T16:13:43,738 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=1a7c5470b472,44715,1732378420375 in 373 msec 2024-11-23T16:13:43,745 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-23T16:13:43,745 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 585 msec 2024-11-23T16:13:43,747 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-23T16:13:43,747 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-23T16:13:43,771 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T16:13:43,773 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1a7c5470b472,44715,1732378420375, seqNum=-1] 2024-11-23T16:13:43,795 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T16:13:43,798 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48207, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T16:13:43,822 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.2610 sec 2024-11-23T16:13:43,822 INFO [master/1a7c5470b472:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732378423822, completionTime=-1 2024-11-23T16:13:43,826 INFO [master/1a7c5470b472:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-11-23T16:13:43,826 DEBUG [master/1a7c5470b472:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-23T16:13:43,877 INFO [master/1a7c5470b472:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-11-23T16:13:43,877 INFO [master/1a7c5470b472:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732378483877 2024-11-23T16:13:43,877 INFO [master/1a7c5470b472:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732378543877 2024-11-23T16:13:43,878 INFO [master/1a7c5470b472:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 51 msec 2024-11-23T16:13:43,880 INFO [master/1a7c5470b472:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=1a7c5470b472,45799,1732378419617-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-23T16:13:43,880 INFO [master/1a7c5470b472:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=1a7c5470b472,45799,1732378419617-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-23T16:13:43,880 INFO [master/1a7c5470b472:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=1a7c5470b472,45799,1732378419617-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-23T16:13:43,882 INFO [master/1a7c5470b472:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-1a7c5470b472:45799, period=300000, unit=MILLISECONDS is enabled. 2024-11-23T16:13:43,883 INFO [master/1a7c5470b472:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-23T16:13:43,884 INFO [master/1a7c5470b472:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-23T16:13:43,902 DEBUG [master/1a7c5470b472:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-23T16:13:43,929 INFO [master/1a7c5470b472:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 3.407sec 2024-11-23T16:13:43,930 INFO [master/1a7c5470b472:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-23T16:13:43,931 INFO [master/1a7c5470b472:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-23T16:13:43,932 INFO [master/1a7c5470b472:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-23T16:13:43,933 INFO [master/1a7c5470b472:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-23T16:13:43,933 INFO [master/1a7c5470b472:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-23T16:13:43,934 INFO [master/1a7c5470b472:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=1a7c5470b472,45799,1732378419617-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-23T16:13:43,935 INFO [master/1a7c5470b472:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=1a7c5470b472,45799,1732378419617-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-23T16:13:43,944 DEBUG [master/1a7c5470b472:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-23T16:13:43,946 INFO [master/1a7c5470b472:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-23T16:13:43,946 INFO [master/1a7c5470b472:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=1a7c5470b472,45799,1732378419617-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-23T16:13:43,998 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@343aff6a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T16:13:44,001 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-23T16:13:44,001 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-23T16:13:44,006 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1a7c5470b472,45799,-1 for getting cluster id 2024-11-23T16:13:44,009 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-23T16:13:44,021 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '34e0592b-548a-4d73-be6a-df1bafadfd92' 2024-11-23T16:13:44,024 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-23T16:13:44,025 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "34e0592b-548a-4d73-be6a-df1bafadfd92" 2024-11-23T16:13:44,028 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2d8c0cff, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T16:13:44,028 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1a7c5470b472,45799,-1] 2024-11-23T16:13:44,031 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-23T16:13:44,037 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T16:13:44,038 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33026, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-23T16:13:44,042 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32372268, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T16:13:44,042 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T16:13:44,051 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1a7c5470b472,44715,1732378420375, seqNum=-1] 2024-11-23T16:13:44,051 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T16:13:44,055 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40322, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T16:13:44,083 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=1a7c5470b472,45799,1732378419617 2024-11-23T16:13:44,105 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMixedAtomicity Thread=211, OpenFileDescriptor=441, MaxFileDescriptor=1048576, SystemLoadAverage=198, ProcessCount=11, AvailableMemoryMB=7806 2024-11-23T16:13:44,117 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-23T16:13:44,123 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 1a7c5470b472,45799,1732378419617 2024-11-23T16:13:44,126 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@61e4ff20 2024-11-23T16:13:44,127 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-23T16:13:44,135 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33028, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-23T16:13:44,137 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45799 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-23T16:13:44,143 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45799 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-23T16:13:44,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45799 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-23T16:13:44,156 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-23T16:13:44,158 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45799 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 4 2024-11-23T16:13:44,159 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:13:44,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-23T16:13:44,165 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-23T16:13:44,179 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073741835_1011 (size=1061) 2024-11-23T16:13:44,192 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 4550087eb34ce466d194f9ce96539302, NAME => 'TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca 2024-11-23T16:13:44,219 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073741836_1012 (size=53) 2024-11-23T16:13:44,220 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T16:13:44,220 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 4550087eb34ce466d194f9ce96539302, disabling compactions & flushes 2024-11-23T16:13:44,221 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302. 2024-11-23T16:13:44,221 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302. 2024-11-23T16:13:44,221 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302. after waiting 0 ms 2024-11-23T16:13:44,221 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302. 2024-11-23T16:13:44,221 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302. 2024-11-23T16:13:44,221 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 4550087eb34ce466d194f9ce96539302: Waiting for close lock at 1732378424220Disabling compacts and flushes for region at 1732378424220Disabling writes for close at 1732378424221 (+1 ms)Writing region close event to WAL at 1732378424221Closed at 1732378424221 2024-11-23T16:13:44,223 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-23T16:13:44,228 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732378424224"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732378424224"}]},"ts":"1732378424224"} 2024-11-23T16:13:44,234 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-23T16:13:44,236 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-23T16:13:44,239 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732378424236"}]},"ts":"1732378424236"} 2024-11-23T16:13:44,244 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-23T16:13:44,246 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=4550087eb34ce466d194f9ce96539302, ASSIGN}] 2024-11-23T16:13:44,249 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=4550087eb34ce466d194f9ce96539302, ASSIGN 2024-11-23T16:13:44,251 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=4550087eb34ce466d194f9ce96539302, ASSIGN; state=OFFLINE, location=1a7c5470b472,44715,1732378420375; forceNewPlan=false, retain=false 2024-11-23T16:13:44,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-23T16:13:44,403 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=4550087eb34ce466d194f9ce96539302, regionState=OPENING, regionLocation=1a7c5470b472,44715,1732378420375 2024-11-23T16:13:44,408 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=4550087eb34ce466d194f9ce96539302, ASSIGN because future has completed 2024-11-23T16:13:44,410 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375}] 2024-11-23T16:13:44,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-23T16:13:44,572 INFO [RS_OPEN_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302. 2024-11-23T16:13:44,572 DEBUG [RS_OPEN_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 4550087eb34ce466d194f9ce96539302, NAME => 'TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302.', STARTKEY => '', ENDKEY => ''} 2024-11-23T16:13:44,573 DEBUG [RS_OPEN_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 4550087eb34ce466d194f9ce96539302 2024-11-23T16:13:44,573 DEBUG [RS_OPEN_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T16:13:44,573 DEBUG [RS_OPEN_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 4550087eb34ce466d194f9ce96539302 2024-11-23T16:13:44,573 DEBUG [RS_OPEN_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 4550087eb34ce466d194f9ce96539302 2024-11-23T16:13:44,575 INFO [StoreOpener-4550087eb34ce466d194f9ce96539302-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 4550087eb34ce466d194f9ce96539302 2024-11-23T16:13:44,578 INFO [StoreOpener-4550087eb34ce466d194f9ce96539302-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-23T16:13:44,579 INFO [StoreOpener-4550087eb34ce466d194f9ce96539302-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4550087eb34ce466d194f9ce96539302 columnFamilyName A 2024-11-23T16:13:44,579 DEBUG [StoreOpener-4550087eb34ce466d194f9ce96539302-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:13:44,580 INFO [StoreOpener-4550087eb34ce466d194f9ce96539302-1 {}] regionserver.HStore(327): Store=4550087eb34ce466d194f9ce96539302/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T16:13:44,580 INFO [StoreOpener-4550087eb34ce466d194f9ce96539302-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 4550087eb34ce466d194f9ce96539302 2024-11-23T16:13:44,582 INFO [StoreOpener-4550087eb34ce466d194f9ce96539302-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-23T16:13:44,582 INFO [StoreOpener-4550087eb34ce466d194f9ce96539302-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4550087eb34ce466d194f9ce96539302 columnFamilyName B 2024-11-23T16:13:44,582 DEBUG [StoreOpener-4550087eb34ce466d194f9ce96539302-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:13:44,583 INFO [StoreOpener-4550087eb34ce466d194f9ce96539302-1 {}] regionserver.HStore(327): Store=4550087eb34ce466d194f9ce96539302/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T16:13:44,583 INFO [StoreOpener-4550087eb34ce466d194f9ce96539302-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 4550087eb34ce466d194f9ce96539302 2024-11-23T16:13:44,585 INFO [StoreOpener-4550087eb34ce466d194f9ce96539302-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-23T16:13:44,585 INFO [StoreOpener-4550087eb34ce466d194f9ce96539302-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4550087eb34ce466d194f9ce96539302 columnFamilyName C 2024-11-23T16:13:44,585 DEBUG [StoreOpener-4550087eb34ce466d194f9ce96539302-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:13:44,586 INFO [StoreOpener-4550087eb34ce466d194f9ce96539302-1 {}] regionserver.HStore(327): Store=4550087eb34ce466d194f9ce96539302/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T16:13:44,587 INFO [RS_OPEN_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302. 2024-11-23T16:13:44,587 DEBUG [RS_OPEN_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 4550087eb34ce466d194f9ce96539302 2024-11-23T16:13:44,588 DEBUG [RS_OPEN_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302 2024-11-23T16:13:44,589 DEBUG [RS_OPEN_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302 2024-11-23T16:13:44,591 DEBUG [RS_OPEN_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 4550087eb34ce466d194f9ce96539302 2024-11-23T16:13:44,591 DEBUG [RS_OPEN_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 4550087eb34ce466d194f9ce96539302 2024-11-23T16:13:44,592 DEBUG [RS_OPEN_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-23T16:13:44,595 DEBUG [RS_OPEN_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 4550087eb34ce466d194f9ce96539302 2024-11-23T16:13:44,599 DEBUG [RS_OPEN_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-23T16:13:44,600 INFO [RS_OPEN_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 4550087eb34ce466d194f9ce96539302; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70214450, jitterRate=0.04627683758735657}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-23T16:13:44,600 DEBUG [RS_OPEN_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 4550087eb34ce466d194f9ce96539302 2024-11-23T16:13:44,601 DEBUG [RS_OPEN_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 4550087eb34ce466d194f9ce96539302: Running coprocessor pre-open hook at 1732378424573Writing region info on filesystem at 1732378424573Initializing all the Stores at 1732378424575 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732378424575Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732378424575Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732378424575Cleaning up temporary data from old regions at 1732378424591 (+16 ms)Running coprocessor post-open hooks at 1732378424600 (+9 ms)Region opened successfully at 1732378424600 2024-11-23T16:13:44,603 INFO [RS_OPEN_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., pid=6, masterSystemTime=1732378424564 2024-11-23T16:13:44,606 DEBUG [RS_OPEN_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302. 2024-11-23T16:13:44,606 INFO [RS_OPEN_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302. 2024-11-23T16:13:44,607 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=4550087eb34ce466d194f9ce96539302, regionState=OPEN, openSeqNum=2, regionLocation=1a7c5470b472,44715,1732378420375 2024-11-23T16:13:44,611 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 because future has completed 2024-11-23T16:13:44,616 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-23T16:13:44,616 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 in 203 msec 2024-11-23T16:13:44,620 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-23T16:13:44,620 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=4550087eb34ce466d194f9ce96539302, ASSIGN in 370 msec 2024-11-23T16:13:44,622 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-23T16:13:44,622 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732378424622"}]},"ts":"1732378424622"} 2024-11-23T16:13:44,625 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-23T16:13:44,626 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-23T16:13:44,629 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 480 msec 2024-11-23T16:13:44,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-23T16:13:44,800 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-23T16:13:44,805 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2ad452c3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T16:13:44,805 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1a7c5470b472,45799,-1 for getting cluster id 2024-11-23T16:13:44,805 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-23T16:13:44,807 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '34e0592b-548a-4d73-be6a-df1bafadfd92' 2024-11-23T16:13:44,807 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-23T16:13:44,807 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "34e0592b-548a-4d73-be6a-df1bafadfd92" 2024-11-23T16:13:44,808 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@20fa5eed, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T16:13:44,808 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1a7c5470b472,45799,-1] 2024-11-23T16:13:44,808 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-23T16:13:44,808 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T16:13:44,810 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33046, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-23T16:13:44,811 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a66f173, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T16:13:44,812 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T16:13:44,813 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1a7c5470b472,44715,1732378420375, seqNum=-1] 2024-11-23T16:13:44,814 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T16:13:44,816 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40336, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T16:13:44,818 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-23T16:13:44,820 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 1a7c5470b472,45799,1732378419617 2024-11-23T16:13:44,820 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@1c52dcb8 2024-11-23T16:13:44,823 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-23T16:13:44,825 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33056, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-23T16:13:44,833 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19d419ec, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T16:13:44,833 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1a7c5470b472,45799,-1 for getting cluster id 2024-11-23T16:13:44,833 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-23T16:13:44,834 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '34e0592b-548a-4d73-be6a-df1bafadfd92' 2024-11-23T16:13:44,835 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-23T16:13:44,835 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "34e0592b-548a-4d73-be6a-df1bafadfd92" 2024-11-23T16:13:44,836 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@21d7efe6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T16:13:44,836 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1a7c5470b472,45799,-1] 2024-11-23T16:13:44,836 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-23T16:13:44,836 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T16:13:44,838 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33074, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-23T16:13:44,839 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7230d8e9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T16:13:44,841 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@130d4a2a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T16:13:44,841 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1a7c5470b472,45799,-1 for getting cluster id 2024-11-23T16:13:44,841 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-23T16:13:44,845 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '34e0592b-548a-4d73-be6a-df1bafadfd92' 2024-11-23T16:13:44,846 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-23T16:13:44,846 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "34e0592b-548a-4d73-be6a-df1bafadfd92" 2024-11-23T16:13:44,846 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@553a6f47, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T16:13:44,846 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1a7c5470b472,45799,-1] 2024-11-23T16:13:44,847 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-23T16:13:44,848 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T16:13:44,851 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33096, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-23T16:13:44,853 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7fc4f8ed, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T16:13:44,854 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@59d35085, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T16:13:44,854 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1a7c5470b472,45799,-1 for getting cluster id 2024-11-23T16:13:44,854 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-23T16:13:44,857 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '34e0592b-548a-4d73-be6a-df1bafadfd92' 2024-11-23T16:13:44,858 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-23T16:13:44,858 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "34e0592b-548a-4d73-be6a-df1bafadfd92" 2024-11-23T16:13:44,859 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@14ba4844, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T16:13:44,859 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1a7c5470b472,45799,-1] 2024-11-23T16:13:44,859 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-23T16:13:44,860 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T16:13:44,861 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33124, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-23T16:13:44,862 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c5b0754, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T16:13:44,863 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3f00f692, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T16:13:44,863 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1a7c5470b472,45799,-1 for getting cluster id 2024-11-23T16:13:44,863 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-23T16:13:44,865 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '34e0592b-548a-4d73-be6a-df1bafadfd92' 2024-11-23T16:13:44,865 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-23T16:13:44,866 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "34e0592b-548a-4d73-be6a-df1bafadfd92" 2024-11-23T16:13:44,866 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@22e855a7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T16:13:44,866 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1a7c5470b472,45799,-1] 2024-11-23T16:13:44,866 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-23T16:13:44,867 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T16:13:44,868 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33146, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-23T16:13:44,870 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7b24d866, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T16:13:44,871 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@388c382, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T16:13:44,871 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1a7c5470b472,45799,-1 for getting cluster id 2024-11-23T16:13:44,871 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-23T16:13:44,872 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '34e0592b-548a-4d73-be6a-df1bafadfd92' 2024-11-23T16:13:44,872 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-23T16:13:44,872 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "34e0592b-548a-4d73-be6a-df1bafadfd92" 2024-11-23T16:13:44,873 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@39fcd423, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T16:13:44,873 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1a7c5470b472,45799,-1] 2024-11-23T16:13:44,873 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-23T16:13:44,873 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T16:13:44,874 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33174, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-23T16:13:44,875 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4787ae6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T16:13:44,878 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7ff89310, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T16:13:44,878 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1a7c5470b472,45799,-1 for getting cluster id 2024-11-23T16:13:44,879 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-23T16:13:44,880 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '34e0592b-548a-4d73-be6a-df1bafadfd92' 2024-11-23T16:13:44,880 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-23T16:13:44,880 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "34e0592b-548a-4d73-be6a-df1bafadfd92" 2024-11-23T16:13:44,881 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@49373cb6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T16:13:44,881 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1a7c5470b472,45799,-1] 2024-11-23T16:13:44,881 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-23T16:13:44,882 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T16:13:44,882 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33180, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-23T16:13:44,883 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6a23129e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T16:13:44,884 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@761fbe54, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T16:13:44,884 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1a7c5470b472,45799,-1 for getting cluster id 2024-11-23T16:13:44,885 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-23T16:13:44,886 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '34e0592b-548a-4d73-be6a-df1bafadfd92' 2024-11-23T16:13:44,886 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-23T16:13:44,886 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "34e0592b-548a-4d73-be6a-df1bafadfd92" 2024-11-23T16:13:44,886 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4f83cf40, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T16:13:44,886 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1a7c5470b472,45799,-1] 2024-11-23T16:13:44,887 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-23T16:13:44,887 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T16:13:44,888 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33192, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-23T16:13:44,889 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@605ecfde, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T16:13:44,890 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2019371a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T16:13:44,891 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1a7c5470b472,45799,-1 for getting cluster id 2024-11-23T16:13:44,891 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-23T16:13:44,892 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '34e0592b-548a-4d73-be6a-df1bafadfd92' 2024-11-23T16:13:44,892 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-23T16:13:44,892 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "34e0592b-548a-4d73-be6a-df1bafadfd92" 2024-11-23T16:13:44,893 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5b12d099, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T16:13:44,893 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1a7c5470b472,45799,-1] 2024-11-23T16:13:44,893 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-23T16:13:44,893 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T16:13:44,897 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33218, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-23T16:13:44,898 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@492f9788, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T16:13:44,905 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@36df70c8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T16:13:44,905 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1a7c5470b472,45799,-1 for getting cluster id 2024-11-23T16:13:44,905 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-23T16:13:44,906 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '34e0592b-548a-4d73-be6a-df1bafadfd92' 2024-11-23T16:13:44,907 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-23T16:13:44,907 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "34e0592b-548a-4d73-be6a-df1bafadfd92" 2024-11-23T16:13:44,907 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@684b04c4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T16:13:44,907 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1a7c5470b472,45799,-1] 2024-11-23T16:13:44,908 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-23T16:13:44,908 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T16:13:44,909 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33230, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-23T16:13:44,911 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@27d87de4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T16:13:44,917 DEBUG [Thread-145 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T16:13:44,917 DEBUG [Thread-156 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T16:13:44,919 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1a7c5470b472,44715,1732378420375, seqNum=-1] 2024-11-23T16:13:44,919 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1a7c5470b472,44715,1732378420375, seqNum=-1] 2024-11-23T16:13:44,920 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T16:13:44,920 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T16:13:44,921 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40366, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T16:13:44,922 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40350, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T16:13:44,930 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2] 2024-11-23T16:13:44,931 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2] 2024-11-23T16:13:44,934 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45799 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-23T16:13:44,935 DEBUG [Thread-143 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T16:13:44,937 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1a7c5470b472,44715,1732378420375, seqNum=-1] 2024-11-23T16:13:44,938 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T16:13:44,940 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40380, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T16:13:44,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45799 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees 2024-11-23T16:13:44,944 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-23T16:13:44,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-23T16:13:44,945 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2] 2024-11-23T16:13:44,948 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-23T16:13:44,949 DEBUG [Thread-149 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T16:13:44,950 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-23T16:13:44,951 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1a7c5470b472,44715,1732378420375, seqNum=-1] 2024-11-23T16:13:44,951 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T16:13:44,953 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40390, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T16:13:44,956 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2] 2024-11-23T16:13:44,958 DEBUG [Thread-160 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T16:13:44,959 DEBUG [Thread-147 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T16:13:44,961 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1a7c5470b472,44715,1732378420375, seqNum=-1] 2024-11-23T16:13:44,961 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T16:13:44,963 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40404, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T16:13:44,963 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1a7c5470b472,44715,1732378420375, seqNum=-1] 2024-11-23T16:13:44,963 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T16:13:44,965 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40410, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T16:13:44,967 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2] 2024-11-23T16:13:44,969 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2] 2024-11-23T16:13:44,973 DEBUG [Thread-151 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T16:13:44,975 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1a7c5470b472,44715,1732378420375, seqNum=-1] 2024-11-23T16:13:44,975 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T16:13:44,977 DEBUG [Thread-154 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T16:13:44,977 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40416, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T16:13:44,980 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2] 2024-11-23T16:13:44,980 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1a7c5470b472,44715,1732378420375, seqNum=-1] 2024-11-23T16:13:44,981 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T16:13:44,983 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40418, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T16:13:44,987 DEBUG [Thread-158 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T16:13:44,988 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2] 2024-11-23T16:13:44,990 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1a7c5470b472,44715,1732378420375, seqNum=-1] 2024-11-23T16:13:44,990 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T16:13:44,991 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40432, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T16:13:44,994 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2] 2024-11-23T16:13:45,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(8855): Flush requested on 4550087eb34ce466d194f9ce96539302 2024-11-23T16:13:45,021 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4550087eb34ce466d194f9ce96539302 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-23T16:13:45,033 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4550087eb34ce466d194f9ce96539302, store=A 2024-11-23T16:13:45,034 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-23T16:13:45,036 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4550087eb34ce466d194f9ce96539302, store=B 2024-11-23T16:13:45,036 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-23T16:13:45,036 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4550087eb34ce466d194f9ce96539302, store=C 2024-11-23T16:13:45,036 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-23T16:13:45,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-23T16:13:45,130 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-23T16:13:45,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302. 2024-11-23T16:13:45,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302. as already flushing 2024-11-23T16:13:45,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302. 2024-11-23T16:13:45,147 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 4550087eb34ce466d194f9ce96539302, NAME => 'TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:13:45,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 4550087eb34ce466d194f9ce96539302, NAME => 'TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:13:45,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45799 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4550087eb34ce466d194f9ce96539302, NAME => 'TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4550087eb34ce466d194f9ce96539302, NAME => 'TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:13:45,182 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp/A/11d8100dbeac4ff3b5ef044fae4ffa8c is 50, key is test_row_0/A:col10/1732378425005/Put/seqid=0 2024-11-23T16:13:45,199 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073741837_1013 (size=12001) 2024-11-23T16:13:45,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-23T16:13:45,274 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:13:45,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40390 deadline: 1732378485246, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:13:45,279 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1a7c5470b472:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732378425137, max timestamp=1732378425257Num uniques 859; 2024-11-23T16:13:45,280 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:13:45,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40350 deadline: 1732378485248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:13:45,281 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1a7c5470b472:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732378425137, max timestamp=1732378425257Num uniques 859; 2024-11-23T16:13:45,281 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1a7c5470b472:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-23T16:13:45,282 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:45,282 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:45,281 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1a7c5470b472:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732378425137, max timestamp=1732378425257Num uniques 859; 2024-11-23T16:13:45,284 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:45,284 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1a7c5470b472:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-23T16:13:45,284 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:45,284 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:13:45,284 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:13:45,281 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1a7c5470b472:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-23T16:13:45,322 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-23T16:13:45,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302. 2024-11-23T16:13:45,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302. as already flushing 2024-11-23T16:13:45,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302. 2024-11-23T16:13:45,339 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 4550087eb34ce466d194f9ce96539302, NAME => 'TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:13:45,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 4550087eb34ce466d194f9ce96539302, NAME => 'TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:13:45,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45799 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4550087eb34ce466d194f9ce96539302, NAME => 'TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4550087eb34ce466d194f9ce96539302, NAME => 'TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:13:45,378 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:13:45,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40416 deadline: 1732378485363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:13:45,380 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:45,380 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:45,381 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:13:45,380 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:13:45,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40410 deadline: 1732378485367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:13:45,382 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:45,383 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:45,383 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:13:45,388 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:13:45,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40380 deadline: 1732378485369, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:13:45,392 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:45,392 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:45,393 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:13:45,393 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:13:45,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40350 deadline: 1732378485389, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:13:45,394 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:45,395 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:45,395 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:13:45,395 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:13:45,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40390 deadline: 1732378485390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:13:45,396 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:45,396 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:45,397 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:13:45,495 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:13:45,495 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:13:45,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40416 deadline: 1732378485490, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:13:45,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40410 deadline: 1732378485491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:13:45,496 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-23T16:13:45,497 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:45,497 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:45,497 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:45,497 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:13:45,497 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:45,498 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:13:45,503 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:13:45,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40380 deadline: 1732378485499, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:13:45,505 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:45,505 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:45,505 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:13:45,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302. 2024-11-23T16:13:45,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302. as already flushing 2024-11-23T16:13:45,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302. 2024-11-23T16:13:45,517 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 4550087eb34ce466d194f9ce96539302, NAME => 'TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:13:45,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 4550087eb34ce466d194f9ce96539302, NAME => 'TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:13:45,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45799 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4550087eb34ce466d194f9ce96539302, NAME => 'TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4550087eb34ce466d194f9ce96539302, NAME => 'TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:13:45,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-23T16:13:45,601 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp/A/11d8100dbeac4ff3b5ef044fae4ffa8c 2024-11-23T16:13:45,604 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:13:45,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40350 deadline: 1732378485599, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:13:45,608 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:45,609 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:45,609 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:13:45,610 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:13:45,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40390 deadline: 1732378485608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:13:45,611 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:45,611 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:45,612 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:13:45,673 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-23T16:13:45,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302. 2024-11-23T16:13:45,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302. as already flushing 2024-11-23T16:13:45,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302. 2024-11-23T16:13:45,674 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 4550087eb34ce466d194f9ce96539302, NAME => 'TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:13:45,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 4550087eb34ce466d194f9ce96539302, NAME => 'TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:13:45,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45799 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4550087eb34ce466d194f9ce96539302, NAME => 'TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4550087eb34ce466d194f9ce96539302, NAME => 'TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:13:45,714 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:13:45,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40416 deadline: 1732378485709, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:13:45,715 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:45,715 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:45,715 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:13:45,716 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:13:45,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40410 deadline: 1732378485709, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:13:45,717 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:45,717 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:45,717 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:13:45,721 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:13:45,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40380 deadline: 1732378485710, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:13:45,727 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:45,727 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:45,727 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:13:45,741 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp/B/3a20a59de3934a8084afbc6412ec85d2 is 50, key is test_row_0/B:col10/1732378425005/Put/seqid=0 2024-11-23T16:13:45,769 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073741838_1014 (size=12001) 2024-11-23T16:13:45,774 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp/B/3a20a59de3934a8084afbc6412ec85d2 2024-11-23T16:13:45,826 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp/C/1883edb4efeb459d8bff1bd5d8269417 is 50, key is test_row_0/C:col10/1732378425005/Put/seqid=0 2024-11-23T16:13:45,830 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-23T16:13:45,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302. 2024-11-23T16:13:45,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302. as already flushing 2024-11-23T16:13:45,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302. 2024-11-23T16:13:45,831 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 4550087eb34ce466d194f9ce96539302, NAME => 'TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:13:45,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 4550087eb34ce466d194f9ce96539302, NAME => 'TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:13:45,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45799 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4550087eb34ce466d194f9ce96539302, NAME => 'TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4550087eb34ce466d194f9ce96539302, NAME => 'TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:13:45,883 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073741839_1015 (size=12001) 2024-11-23T16:13:45,921 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:13:45,921 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:13:45,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40390 deadline: 1732378485919, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:13:45,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40350 deadline: 1732378485919, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:13:45,923 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:45,923 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:45,923 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:45,923 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:13:45,923 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:45,923 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:13:45,989 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-23T16:13:45,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302. 2024-11-23T16:13:45,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302. as already flushing 2024-11-23T16:13:45,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302. 2024-11-23T16:13:45,991 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 4550087eb34ce466d194f9ce96539302, NAME => 'TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:13:45,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 4550087eb34ce466d194f9ce96539302, NAME => 'TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:13:45,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45799 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4550087eb34ce466d194f9ce96539302, NAME => 'TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4550087eb34ce466d194f9ce96539302, NAME => 'TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:13:46,033 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:13:46,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40410 deadline: 1732378486029, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:13:46,034 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:46,034 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:46,035 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:13:46,035 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:13:46,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40416 deadline: 1732378486029, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:13:46,036 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:46,036 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:46,036 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:13:46,043 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:13:46,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40380 deadline: 1732378486039, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:13:46,047 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:46,047 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:46,047 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:13:46,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-23T16:13:46,147 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-23T16:13:46,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302. 2024-11-23T16:13:46,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302. as already flushing 2024-11-23T16:13:46,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302. 2024-11-23T16:13:46,154 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 4550087eb34ce466d194f9ce96539302, NAME => 'TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:13:46,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 4550087eb34ce466d194f9ce96539302, NAME => 'TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:13:46,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45799 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4550087eb34ce466d194f9ce96539302, NAME => 'TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4550087eb34ce466d194f9ce96539302, NAME => 'TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:13:46,285 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp/C/1883edb4efeb459d8bff1bd5d8269417 2024-11-23T16:13:46,302 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp/A/11d8100dbeac4ff3b5ef044fae4ffa8c as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/A/11d8100dbeac4ff3b5ef044fae4ffa8c 2024-11-23T16:13:46,311 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-23T16:13:46,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302. 2024-11-23T16:13:46,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302. as already flushing 2024-11-23T16:13:46,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302. 2024-11-23T16:13:46,317 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 4550087eb34ce466d194f9ce96539302, NAME => 'TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:13:46,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 4550087eb34ce466d194f9ce96539302, NAME => 'TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:13:46,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45799 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4550087eb34ce466d194f9ce96539302, NAME => 'TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4550087eb34ce466d194f9ce96539302, NAME => 'TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:13:46,322 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/A/11d8100dbeac4ff3b5ef044fae4ffa8c, entries=150, sequenceid=12, filesize=11.7 K 2024-11-23T16:13:46,327 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreScanner(1005): StoreScanner already closing. There is no need to updateReaders 2024-11-23T16:13:46,330 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp/B/3a20a59de3934a8084afbc6412ec85d2 as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/B/3a20a59de3934a8084afbc6412ec85d2 2024-11-23T16:13:46,345 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/B/3a20a59de3934a8084afbc6412ec85d2, entries=150, sequenceid=12, filesize=11.7 K 2024-11-23T16:13:46,347 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp/C/1883edb4efeb459d8bff1bd5d8269417 as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/C/1883edb4efeb459d8bff1bd5d8269417 2024-11-23T16:13:46,358 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/C/1883edb4efeb459d8bff1bd5d8269417, entries=150, sequenceid=12, filesize=11.7 K 2024-11-23T16:13:46,366 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=134.18 KB/137400 for 4550087eb34ce466d194f9ce96539302 in 1344ms, sequenceid=12, compaction requested=false 2024-11-23T16:13:46,367 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-23T16:13:46,370 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4550087eb34ce466d194f9ce96539302: 2024-11-23T16:13:46,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(8855): Flush requested on 4550087eb34ce466d194f9ce96539302 2024-11-23T16:13:46,449 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4550087eb34ce466d194f9ce96539302 3/3 column families, dataSize=140.89 KB heapSize=400.45 KB 2024-11-23T16:13:46,472 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-23T16:13:46,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302. 2024-11-23T16:13:46,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302. as already flushing 2024-11-23T16:13:46,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302. 2024-11-23T16:13:46,473 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 4550087eb34ce466d194f9ce96539302, NAME => 'TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:13:46,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 4550087eb34ce466d194f9ce96539302, NAME => 'TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:13:46,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45799 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4550087eb34ce466d194f9ce96539302, NAME => 'TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4550087eb34ce466d194f9ce96539302, NAME => 'TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:13:46,493 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4550087eb34ce466d194f9ce96539302, store=A 2024-11-23T16:13:46,496 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1a7c5470b472:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1732378425361, max timestamp=1732378426443Num uniques 1068; 2024-11-23T16:13:46,496 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1a7c5470b472:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=7688, suffix heap size=139040,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=168 2024-11-23T16:13:46,501 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-23T16:13:46,501 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4550087eb34ce466d194f9ce96539302, store=B 2024-11-23T16:13:46,501 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-23T16:13:46,501 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4550087eb34ce466d194f9ce96539302, store=C 2024-11-23T16:13:46,501 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-23T16:13:46,530 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp/A/78d5b357a7364ecfab262ea4b8c8dbb8 is 50, key is test_row_0/A:col10/1732378426442/Put/seqid=0 2024-11-23T16:13:46,587 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073741840_1016 (size=14341) 2024-11-23T16:13:46,628 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-23T16:13:46,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302. 2024-11-23T16:13:46,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302. as already flushing 2024-11-23T16:13:46,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302. 2024-11-23T16:13:46,629 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 4550087eb34ce466d194f9ce96539302, NAME => 'TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:13:46,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 4550087eb34ce466d194f9ce96539302, NAME => 'TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:13:46,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45799 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4550087eb34ce466d194f9ce96539302, NAME => 'TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4550087eb34ce466d194f9ce96539302, NAME => 'TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:13:46,675 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:13:46,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40390 deadline: 1732378486637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:13:46,676 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:46,676 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:13:46,676 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:46,677 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:13:46,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40410 deadline: 1732378486644, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:13:46,677 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:46,677 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:46,677 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:13:46,678 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:13:46,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40416 deadline: 1732378486647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:13:46,679 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:13:46,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40350 deadline: 1732378486653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:13:46,679 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:13:46,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40380 deadline: 1732378486654, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:13:46,680 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:46,680 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:46,680 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:13:46,680 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:46,680 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:46,680 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:13:46,681 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:46,681 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:46,681 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:13:46,784 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-23T16:13:46,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302. 2024-11-23T16:13:46,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302. as already flushing 2024-11-23T16:13:46,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302. 2024-11-23T16:13:46,785 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 4550087eb34ce466d194f9ce96539302, NAME => 'TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:13:46,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 4550087eb34ce466d194f9ce96539302, NAME => 'TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:13:46,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45799 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4550087eb34ce466d194f9ce96539302, NAME => 'TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4550087eb34ce466d194f9ce96539302, NAME => 'TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:13:46,790 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:13:46,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40350 deadline: 1732378486789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:13:46,792 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:46,792 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:46,792 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:13:46,792 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:13:46,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40416 deadline: 1732378486790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:13:46,793 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:46,793 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:13:46,793 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:46,793 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:13:46,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40390 deadline: 1732378486790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:13:46,794 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:46,794 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:46,794 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:13:46,794 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:13:46,794 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:13:46,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40410 deadline: 1732378486790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:13:46,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40380 deadline: 1732378486791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:13:46,794 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:46,795 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:46,795 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:13:46,795 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:46,795 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:46,795 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:13:46,942 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-23T16:13:46,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302. 2024-11-23T16:13:46,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302. as already flushing 2024-11-23T16:13:46,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302. 2024-11-23T16:13:46,942 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 4550087eb34ce466d194f9ce96539302, NAME => 'TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:13:46,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 4550087eb34ce466d194f9ce96539302, NAME => 'TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:13:46,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45799 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4550087eb34ce466d194f9ce96539302, NAME => 'TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4550087eb34ce466d194f9ce96539302, NAME => 'TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:13:46,989 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp/A/78d5b357a7364ecfab262ea4b8c8dbb8 2024-11-23T16:13:47,005 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:13:47,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40350 deadline: 1732378486999, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:13:47,006 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:47,006 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:47,007 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:13:47,007 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:13:47,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40416 deadline: 1732378487000, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:13:47,008 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:47,008 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:47,008 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:13:47,008 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:13:47,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40410 deadline: 1732378487000, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:13:47,009 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:47,009 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:13:47,009 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:47,009 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:13:47,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40390 deadline: 1732378487000, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:13:47,010 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:47,010 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:47,010 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:13:47,017 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:13:47,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40380 deadline: 1732378487009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:13:47,018 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:47,018 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:47,018 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:13:47,025 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp/B/94094598c73f4c51bb19675082146305 is 50, key is test_row_0/B:col10/1732378426442/Put/seqid=0 2024-11-23T16:13:47,048 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073741841_1017 (size=12001) 2024-11-23T16:13:47,050 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp/B/94094598c73f4c51bb19675082146305 2024-11-23T16:13:47,073 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp/C/fba9de58dfa84fe389f0869ffb0b8a5e is 50, key is test_row_0/C:col10/1732378426442/Put/seqid=0 2024-11-23T16:13:47,098 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-23T16:13:47,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302. 2024-11-23T16:13:47,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302. as already flushing 2024-11-23T16:13:47,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302. 2024-11-23T16:13:47,099 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 4550087eb34ce466d194f9ce96539302, NAME => 'TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:13:47,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 4550087eb34ce466d194f9ce96539302, NAME => 'TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:13:47,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45799 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4550087eb34ce466d194f9ce96539302, NAME => 'TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4550087eb34ce466d194f9ce96539302, NAME => 'TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:13:47,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-23T16:13:47,110 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073741842_1018 (size=12001) 2024-11-23T16:13:47,117 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp/C/fba9de58dfa84fe389f0869ffb0b8a5e 2024-11-23T16:13:47,132 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp/A/78d5b357a7364ecfab262ea4b8c8dbb8 as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/A/78d5b357a7364ecfab262ea4b8c8dbb8 2024-11-23T16:13:47,150 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/A/78d5b357a7364ecfab262ea4b8c8dbb8, entries=200, sequenceid=50, filesize=14.0 K 2024-11-23T16:13:47,152 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp/B/94094598c73f4c51bb19675082146305 as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/B/94094598c73f4c51bb19675082146305 2024-11-23T16:13:47,165 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/B/94094598c73f4c51bb19675082146305, entries=150, sequenceid=50, filesize=11.7 K 2024-11-23T16:13:47,167 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp/C/fba9de58dfa84fe389f0869ffb0b8a5e as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/C/fba9de58dfa84fe389f0869ffb0b8a5e 2024-11-23T16:13:47,186 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/C/fba9de58dfa84fe389f0869ffb0b8a5e, entries=150, sequenceid=50, filesize=11.7 K 2024-11-23T16:13:47,188 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~302.31 KB/309568, currentSize=80.51 KB/82440 for 4550087eb34ce466d194f9ce96539302 in 739ms, sequenceid=50, compaction requested=false 2024-11-23T16:13:47,188 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4550087eb34ce466d194f9ce96539302: 2024-11-23T16:13:47,254 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-23T16:13:47,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302. 2024-11-23T16:13:47,255 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2902): Flushing 4550087eb34ce466d194f9ce96539302 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-23T16:13:47,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4550087eb34ce466d194f9ce96539302, store=A 2024-11-23T16:13:47,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-23T16:13:47,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4550087eb34ce466d194f9ce96539302, store=B 2024-11-23T16:13:47,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-23T16:13:47,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4550087eb34ce466d194f9ce96539302, store=C 2024-11-23T16:13:47,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-23T16:13:47,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp/A/f65beed7b7e145bf9b59699554bd853f is 50, key is test_row_0/A:col10/1732378426645/Put/seqid=0 2024-11-23T16:13:47,312 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073741843_1019 (size=12001) 2024-11-23T16:13:47,323 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=65 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp/A/f65beed7b7e145bf9b59699554bd853f 2024-11-23T16:13:47,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(8855): Flush requested on 4550087eb34ce466d194f9ce96539302 2024-11-23T16:13:47,330 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302. as already flushing 2024-11-23T16:13:47,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp/B/db978ec1534a4f21bbb8f82b9e8aecdb is 50, key is test_row_0/B:col10/1732378426645/Put/seqid=0 2024-11-23T16:13:47,380 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073741844_1020 (size=12001) 2024-11-23T16:13:47,381 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:13:47,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40380 deadline: 1732378487365, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:13:47,382 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=65 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp/B/db978ec1534a4f21bbb8f82b9e8aecdb 2024-11-23T16:13:47,382 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:47,382 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:47,383 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:13:47,387 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:13:47,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40390 deadline: 1732378487371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:13:47,388 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:47,388 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:47,388 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:13:47,389 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:13:47,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40350 deadline: 1732378487373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:13:47,390 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:47,390 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:47,390 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:13:47,391 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:13:47,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40410 deadline: 1732378487382, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:13:47,391 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:47,391 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:47,392 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:13:47,392 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:13:47,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40416 deadline: 1732378487382, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:13:47,393 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:47,393 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:47,393 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:13:47,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp/C/85bd609f56dd4d4891aaba05ecafd851 is 50, key is test_row_0/C:col10/1732378426645/Put/seqid=0 2024-11-23T16:13:47,414 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073741845_1021 (size=12001) 2024-11-23T16:13:47,416 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=65 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp/C/85bd609f56dd4d4891aaba05ecafd851 2024-11-23T16:13:47,431 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp/A/f65beed7b7e145bf9b59699554bd853f as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/A/f65beed7b7e145bf9b59699554bd853f 2024-11-23T16:13:47,445 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/A/f65beed7b7e145bf9b59699554bd853f, entries=150, sequenceid=65, filesize=11.7 K 2024-11-23T16:13:47,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp/B/db978ec1534a4f21bbb8f82b9e8aecdb as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/B/db978ec1534a4f21bbb8f82b9e8aecdb 2024-11-23T16:13:47,469 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/B/db978ec1534a4f21bbb8f82b9e8aecdb, entries=150, sequenceid=65, filesize=11.7 K 2024-11-23T16:13:47,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp/C/85bd609f56dd4d4891aaba05ecafd851 as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/C/85bd609f56dd4d4891aaba05ecafd851 2024-11-23T16:13:47,483 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/C/85bd609f56dd4d4891aaba05ecafd851, entries=150, sequenceid=65, filesize=11.7 K 2024-11-23T16:13:47,485 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 4550087eb34ce466d194f9ce96539302 in 231ms, sequenceid=65, compaction requested=true 2024-11-23T16:13:47,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2603): Flush status journal for 4550087eb34ce466d194f9ce96539302: 2024-11-23T16:13:47,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302. 2024-11-23T16:13:47,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=8 2024-11-23T16:13:47,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45799 {}] master.HMaster(4169): Remote procedure done, pid=8 2024-11-23T16:13:47,494 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-23T16:13:47,494 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.5390 sec 2024-11-23T16:13:47,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(8855): Flush requested on 4550087eb34ce466d194f9ce96539302 2024-11-23T16:13:47,497 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4550087eb34ce466d194f9ce96539302 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-23T16:13:47,497 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4550087eb34ce466d194f9ce96539302, store=A 2024-11-23T16:13:47,497 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-23T16:13:47,497 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4550087eb34ce466d194f9ce96539302, store=B 2024-11-23T16:13:47,497 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-23T16:13:47,497 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4550087eb34ce466d194f9ce96539302, store=C 2024-11-23T16:13:47,498 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-23T16:13:47,499 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees in 2.5600 sec 2024-11-23T16:13:47,516 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp/A/ee9e8b462a504fcf91d03bf8e1d210ca is 50, key is test_row_0/A:col10/1732378427494/Put/seqid=0 2024-11-23T16:13:47,538 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:13:47,538 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:13:47,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40350 deadline: 1732378487523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:13:47,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40410 deadline: 1732378487526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:13:47,539 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:47,539 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:47,539 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:47,539 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:47,540 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:13:47,539 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:13:47,540 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:13:47,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40416 deadline: 1732378487529, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:13:47,540 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:47,540 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:47,540 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:13:47,541 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:13:47,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40390 deadline: 1732378487532, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:13:47,541 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:47,541 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:47,541 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:13:47,541 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:13:47,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40380 deadline: 1732378487533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:13:47,542 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:47,542 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:47,542 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:13:47,547 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073741846_1022 (size=14341) 2024-11-23T16:13:47,548 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=83 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp/A/ee9e8b462a504fcf91d03bf8e1d210ca 2024-11-23T16:13:47,586 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp/B/5b8fa1919b424c4b8c61a999f7b59903 is 50, key is test_row_0/B:col10/1732378427494/Put/seqid=0 2024-11-23T16:13:47,619 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073741847_1023 (size=12001) 2024-11-23T16:13:47,656 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:13:47,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40410 deadline: 1732378487648, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:13:47,658 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:13:47,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40416 deadline: 1732378487650, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:13:47,659 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:13:47,659 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:47,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40390 deadline: 1732378487650, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:13:47,659 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:47,659 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:13:47,659 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:13:47,659 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:47,660 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:47,660 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:47,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40380 deadline: 1732378487650, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:13:47,660 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:47,660 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:13:47,660 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:13:47,660 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:47,660 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:47,660 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:13:47,665 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:13:47,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40350 deadline: 1732378487649, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:13:47,666 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:47,666 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:47,666 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:13:47,870 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:13:47,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40410 deadline: 1732378487868, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:13:47,871 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:47,871 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:47,871 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:13:47,877 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:13:47,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40416 deadline: 1732378487869, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:13:47,879 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:47,879 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:47,879 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:13:47,879 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:13:47,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40380 deadline: 1732378487869, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:13:47,880 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:47,880 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:13:47,880 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:47,880 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:13:47,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40390 deadline: 1732378487869, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:13:47,881 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:47,881 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:47,881 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:13:47,887 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:13:47,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40350 deadline: 1732378487878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:13:47,888 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:47,888 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:47,888 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:13:48,021 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=83 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp/B/5b8fa1919b424c4b8c61a999f7b59903 2024-11-23T16:13:48,041 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp/C/b0310f2db52d4173a306340c0da14e5c is 50, key is test_row_0/C:col10/1732378427494/Put/seqid=0 2024-11-23T16:13:48,055 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073741848_1024 (size=12001) 2024-11-23T16:13:48,180 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:13:48,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40410 deadline: 1732378488179, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:13:48,182 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:48,182 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:48,182 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:13:48,191 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:13:48,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40416 deadline: 1732378488188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:13:48,192 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:48,192 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:13:48,192 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:48,192 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:13:48,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40380 deadline: 1732378488188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:13:48,193 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:48,193 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:48,193 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:13:48,193 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:13:48,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40390 deadline: 1732378488189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:13:48,194 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:48,194 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:48,194 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:13:48,200 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:13:48,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40350 deadline: 1732378488198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:13:48,201 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:48,201 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:48,201 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:13:48,458 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=83 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp/C/b0310f2db52d4173a306340c0da14e5c 2024-11-23T16:13:48,479 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp/A/ee9e8b462a504fcf91d03bf8e1d210ca as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/A/ee9e8b462a504fcf91d03bf8e1d210ca 2024-11-23T16:13:48,495 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/A/ee9e8b462a504fcf91d03bf8e1d210ca, entries=200, sequenceid=83, filesize=14.0 K 2024-11-23T16:13:48,498 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp/B/5b8fa1919b424c4b8c61a999f7b59903 as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/B/5b8fa1919b424c4b8c61a999f7b59903 2024-11-23T16:13:48,519 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/B/5b8fa1919b424c4b8c61a999f7b59903, entries=150, sequenceid=83, filesize=11.7 K 2024-11-23T16:13:48,524 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp/C/b0310f2db52d4173a306340c0da14e5c as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/C/b0310f2db52d4173a306340c0da14e5c 2024-11-23T16:13:48,539 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/C/b0310f2db52d4173a306340c0da14e5c, entries=150, sequenceid=83, filesize=11.7 K 2024-11-23T16:13:48,543 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 4550087eb34ce466d194f9ce96539302 in 1046ms, sequenceid=83, compaction requested=true 2024-11-23T16:13:48,543 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4550087eb34ce466d194f9ce96539302: 2024-11-23T16:13:48,545 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4550087eb34ce466d194f9ce96539302:A, priority=-2147483648, current under compaction store size is 1 2024-11-23T16:13:48,548 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T16:13:48,549 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4550087eb34ce466d194f9ce96539302:B, priority=-2147483648, current under compaction store size is 2 2024-11-23T16:13:48,549 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T16:13:48,549 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4550087eb34ce466d194f9ce96539302:C, priority=-2147483648, current under compaction store size is 3 2024-11-23T16:13:48,549 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T16:13:48,551 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-23T16:13:48,554 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-23T16:13:48,556 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-23T16:13:48,558 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HStore(1541): 4550087eb34ce466d194f9ce96539302/B is initiating minor compaction (all files) 2024-11-23T16:13:48,559 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4550087eb34ce466d194f9ce96539302/B in TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302. 2024-11-23T16:13:48,559 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/B/3a20a59de3934a8084afbc6412ec85d2, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/B/94094598c73f4c51bb19675082146305, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/B/db978ec1534a4f21bbb8f82b9e8aecdb, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/B/5b8fa1919b424c4b8c61a999f7b59903] into tmpdir=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp, totalSize=46.9 K 2024-11-23T16:13:48,561 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.Compactor(225): Compacting 3a20a59de3934a8084afbc6412ec85d2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1732378424982 2024-11-23T16:13:48,565 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52684 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-23T16:13:48,565 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HStore(1541): 4550087eb34ce466d194f9ce96539302/A is initiating minor compaction (all files) 2024-11-23T16:13:48,565 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4550087eb34ce466d194f9ce96539302/A in TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302. 2024-11-23T16:13:48,565 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/A/11d8100dbeac4ff3b5ef044fae4ffa8c, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/A/78d5b357a7364ecfab262ea4b8c8dbb8, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/A/f65beed7b7e145bf9b59699554bd853f, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/A/ee9e8b462a504fcf91d03bf8e1d210ca] into tmpdir=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp, totalSize=51.4 K 2024-11-23T16:13:48,566 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.Compactor(225): Compacting 94094598c73f4c51bb19675082146305, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1732378425364 2024-11-23T16:13:48,566 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.Compactor(225): Compacting 11d8100dbeac4ff3b5ef044fae4ffa8c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1732378424982 2024-11-23T16:13:48,567 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.Compactor(225): Compacting db978ec1534a4f21bbb8f82b9e8aecdb, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=65, earliestPutTs=1732378426556 2024-11-23T16:13:48,568 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.Compactor(225): Compacting 78d5b357a7364ecfab262ea4b8c8dbb8, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1732378425361 2024-11-23T16:13:48,568 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.Compactor(225): Compacting 5b8fa1919b424c4b8c61a999f7b59903, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1732378427360 2024-11-23T16:13:48,569 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.Compactor(225): Compacting f65beed7b7e145bf9b59699554bd853f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=65, earliestPutTs=1732378426556 2024-11-23T16:13:48,572 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.Compactor(225): Compacting ee9e8b462a504fcf91d03bf8e1d210ca, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1732378427360 2024-11-23T16:13:48,641 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4550087eb34ce466d194f9ce96539302#B#compaction#13 average throughput is 0.82 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T16:13:48,642 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp/B/d356752a25ff4bb58f1ef25e56c4ab3b is 50, key is test_row_0/B:col10/1732378427494/Put/seqid=0 2024-11-23T16:13:48,643 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4550087eb34ce466d194f9ce96539302#A#compaction#12 average throughput is 0.73 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T16:13:48,644 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp/A/fc702aca2401483fb69f73fc0cc7511e is 50, key is test_row_0/A:col10/1732378427494/Put/seqid=0 2024-11-23T16:13:48,666 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073741850_1026 (size=12139) 2024-11-23T16:13:48,675 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073741849_1025 (size=12139) 2024-11-23T16:13:48,691 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp/B/d356752a25ff4bb58f1ef25e56c4ab3b as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/B/d356752a25ff4bb58f1ef25e56c4ab3b 2024-11-23T16:13:48,693 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp/A/fc702aca2401483fb69f73fc0cc7511e as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/A/fc702aca2401483fb69f73fc0cc7511e 2024-11-23T16:13:48,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(8855): Flush requested on 4550087eb34ce466d194f9ce96539302 2024-11-23T16:13:48,694 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4550087eb34ce466d194f9ce96539302 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-23T16:13:48,694 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4550087eb34ce466d194f9ce96539302, store=A 2024-11-23T16:13:48,695 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-23T16:13:48,695 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4550087eb34ce466d194f9ce96539302, store=B 2024-11-23T16:13:48,695 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-23T16:13:48,695 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4550087eb34ce466d194f9ce96539302, store=C 2024-11-23T16:13:48,695 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-23T16:13:48,711 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp/A/75298ec17fac429488835c77a3eadcc7 is 50, key is test_row_0/A:col10/1732378427529/Put/seqid=0 2024-11-23T16:13:48,734 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 4550087eb34ce466d194f9ce96539302/B of 4550087eb34ce466d194f9ce96539302 into d356752a25ff4bb58f1ef25e56c4ab3b(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T16:13:48,734 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4550087eb34ce466d194f9ce96539302: 2024-11-23T16:13:48,734 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 4550087eb34ce466d194f9ce96539302/A of 4550087eb34ce466d194f9ce96539302 into fc702aca2401483fb69f73fc0cc7511e(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T16:13:48,734 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4550087eb34ce466d194f9ce96539302: 2024-11-23T16:13:48,737 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073741851_1027 (size=14341) 2024-11-23T16:13:48,738 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=98 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp/A/75298ec17fac429488835c77a3eadcc7 2024-11-23T16:13:48,739 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., storeName=4550087eb34ce466d194f9ce96539302/A, priority=12, startTime=1732378428545; duration=0sec 2024-11-23T16:13:48,739 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., storeName=4550087eb34ce466d194f9ce96539302/B, priority=12, startTime=1732378428548; duration=0sec 2024-11-23T16:13:48,739 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T16:13:48,740 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T16:13:48,740 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4550087eb34ce466d194f9ce96539302:A 2024-11-23T16:13:48,740 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4550087eb34ce466d194f9ce96539302:B 2024-11-23T16:13:48,740 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-23T16:13:48,747 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-23T16:13:48,747 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HStore(1541): 4550087eb34ce466d194f9ce96539302/C is initiating minor compaction (all files) 2024-11-23T16:13:48,747 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4550087eb34ce466d194f9ce96539302/C in TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302. 2024-11-23T16:13:48,748 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/C/1883edb4efeb459d8bff1bd5d8269417, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/C/fba9de58dfa84fe389f0869ffb0b8a5e, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/C/85bd609f56dd4d4891aaba05ecafd851, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/C/b0310f2db52d4173a306340c0da14e5c] into tmpdir=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp, totalSize=46.9 K 2024-11-23T16:13:48,753 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1883edb4efeb459d8bff1bd5d8269417, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1732378424982 2024-11-23T16:13:48,755 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.Compactor(225): Compacting fba9de58dfa84fe389f0869ffb0b8a5e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1732378425364 2024-11-23T16:13:48,756 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.Compactor(225): Compacting 85bd609f56dd4d4891aaba05ecafd851, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=65, earliestPutTs=1732378426556 2024-11-23T16:13:48,757 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.Compactor(225): Compacting b0310f2db52d4173a306340c0da14e5c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1732378427360 2024-11-23T16:13:48,767 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:13:48,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40416 deadline: 1732378488749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:13:48,768 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:13:48,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40350 deadline: 1732378488751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:13:48,769 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:48,769 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:48,769 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:48,769 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:13:48,769 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:48,769 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:13:48,770 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:13:48,770 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp/B/707e1bd9e5814838a99c40af9f3f7468 is 50, key is test_row_0/B:col10/1732378427529/Put/seqid=0 2024-11-23T16:13:48,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40390 deadline: 1732378488753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:13:48,771 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:48,771 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:48,771 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:13:48,771 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:13:48,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40410 deadline: 1732378488759, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:13:48,772 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:13:48,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40380 deadline: 1732378488759, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:13:48,772 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:48,772 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:48,772 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:13:48,772 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:48,773 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:48,773 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:13:48,812 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073741852_1028 (size=12001) 2024-11-23T16:13:48,819 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=98 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp/B/707e1bd9e5814838a99c40af9f3f7468 2024-11-23T16:13:48,829 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4550087eb34ce466d194f9ce96539302#C#compaction#16 average throughput is 0.60 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T16:13:48,833 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp/C/c82cabbae3434cd99b6938c95500884b is 50, key is test_row_0/C:col10/1732378427494/Put/seqid=0 2024-11-23T16:13:48,853 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp/C/4d8a2528d7474e3585dd0e27ab5d5867 is 50, key is test_row_0/C:col10/1732378427529/Put/seqid=0 2024-11-23T16:13:48,863 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073741853_1029 (size=12139) 2024-11-23T16:13:48,885 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:13:48,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40390 deadline: 1732378488879, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:13:48,887 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073741854_1030 (size=12001) 2024-11-23T16:13:48,887 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:48,887 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:48,887 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:13:48,888 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:13:48,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40410 deadline: 1732378488879, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:13:48,888 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=98 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp/C/4d8a2528d7474e3585dd0e27ab5d5867 2024-11-23T16:13:48,888 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:48,888 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:13:48,889 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:48,889 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:13:48,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40416 deadline: 1732378488880, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:13:48,889 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:48,889 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:13:48,889 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:48,890 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:13:48,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40380 deadline: 1732378488880, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:13:48,890 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:48,890 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:13:48,890 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:48,890 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:13:48,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40350 deadline: 1732378488880, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:13:48,891 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:48,891 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:48,891 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:13:48,902 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp/A/75298ec17fac429488835c77a3eadcc7 as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/A/75298ec17fac429488835c77a3eadcc7 2024-11-23T16:13:48,918 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/A/75298ec17fac429488835c77a3eadcc7, entries=200, sequenceid=98, filesize=14.0 K 2024-11-23T16:13:48,922 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp/B/707e1bd9e5814838a99c40af9f3f7468 as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/B/707e1bd9e5814838a99c40af9f3f7468 2024-11-23T16:13:48,933 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/B/707e1bd9e5814838a99c40af9f3f7468, entries=150, sequenceid=98, filesize=11.7 K 2024-11-23T16:13:48,935 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp/C/4d8a2528d7474e3585dd0e27ab5d5867 as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/C/4d8a2528d7474e3585dd0e27ab5d5867 2024-11-23T16:13:48,948 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/C/4d8a2528d7474e3585dd0e27ab5d5867, entries=150, sequenceid=98, filesize=11.7 K 2024-11-23T16:13:48,949 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=100.63 KB/103050 for 4550087eb34ce466d194f9ce96539302 in 255ms, sequenceid=98, compaction requested=false 2024-11-23T16:13:48,949 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4550087eb34ce466d194f9ce96539302: 2024-11-23T16:13:49,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(8855): Flush requested on 4550087eb34ce466d194f9ce96539302 2024-11-23T16:13:49,122 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4550087eb34ce466d194f9ce96539302 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-23T16:13:49,127 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1a7c5470b472:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732378428754, max timestamp=1732378429107Num uniques 859; 2024-11-23T16:13:49,127 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1a7c5470b472:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-23T16:13:49,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-23T16:13:49,128 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-23T16:13:49,129 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45799 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-23T16:13:49,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45799 {}] procedure2.ProcedureExecutor(1139): Stored pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees 2024-11-23T16:13:49,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-23T16:13:49,134 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-23T16:13:49,135 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-23T16:13:49,136 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-23T16:13:49,136 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-23T16:13:49,136 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4550087eb34ce466d194f9ce96539302, store=A 2024-11-23T16:13:49,137 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-23T16:13:49,137 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4550087eb34ce466d194f9ce96539302, store=B 2024-11-23T16:13:49,137 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1a7c5470b472:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1732378428754, max timestamp=1732378429107Num uniques 859; 2024-11-23T16:13:49,137 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-23T16:13:49,137 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4550087eb34ce466d194f9ce96539302, store=C 2024-11-23T16:13:49,137 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-23T16:13:49,138 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1a7c5470b472:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-11-23T16:13:49,159 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp/A/6144aee27cd5447484ee86e1ade2e898 is 50, key is test_row_0/A:col10/1732378429105/Put/seqid=0 2024-11-23T16:13:49,193 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073741855_1031 (size=12001) 2024-11-23T16:13:49,221 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-23T16:13:49,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-23T16:13:49,250 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:13:49,250 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:13:49,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40416 deadline: 1732378489195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:13:49,252 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:49,252 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:49,253 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:13:49,253 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1a7c5470b472:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732378429163, max timestamp=1732378429208Num uniques 859; 2024-11-23T16:13:49,253 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1a7c5470b472:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-23T16:13:49,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40410 deadline: 1732378489189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:13:49,254 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1a7c5470b472:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11854, totalHeapSize=27176, min timestamp=1732378429163, max timestamp=1732378429208Num uniques 859; 2024-11-23T16:13:49,254 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:49,254 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1a7c5470b472:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=11854, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-23T16:13:49,254 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:49,254 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:13:49,280 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp/C/c82cabbae3434cd99b6938c95500884b as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/C/c82cabbae3434cd99b6938c95500884b 2024-11-23T16:13:49,291 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-23T16:13:49,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302. 2024-11-23T16:13:49,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302. as already flushing 2024-11-23T16:13:49,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302. 2024-11-23T16:13:49,291 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 4550087eb34ce466d194f9ce96539302, NAME => 'TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:13:49,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 4550087eb34ce466d194f9ce96539302, NAME => 'TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:13:49,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45799 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4550087eb34ce466d194f9ce96539302, NAME => 'TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4550087eb34ce466d194f9ce96539302, NAME => 'TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:13:49,297 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 4550087eb34ce466d194f9ce96539302/C of 4550087eb34ce466d194f9ce96539302 into c82cabbae3434cd99b6938c95500884b(size=11.9 K), total size for store is 23.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T16:13:49,297 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4550087eb34ce466d194f9ce96539302: 2024-11-23T16:13:49,297 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., storeName=4550087eb34ce466d194f9ce96539302/C, priority=12, startTime=1732378428549; duration=0sec 2024-11-23T16:13:49,298 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T16:13:49,298 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4550087eb34ce466d194f9ce96539302:C 2024-11-23T16:13:49,310 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:13:49,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40380 deadline: 1732378489301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:13:49,312 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:13:49,312 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:49,312 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:49,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40390 deadline: 1732378489303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:13:49,312 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:13:49,311 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:13:49,313 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:49,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40350 deadline: 1732378489302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:13:49,313 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:49,313 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:13:49,313 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:49,313 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:49,313 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:13:49,367 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:13:49,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40416 deadline: 1732378489358, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:13:49,368 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:13:49,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40410 deadline: 1732378489359, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:13:49,370 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:49,370 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:49,370 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:13:49,371 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:49,371 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:49,371 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:13:49,429 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:13:49,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40380 deadline: 1732378489418, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:13:49,430 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:49,430 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:49,430 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:13:49,430 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:13:49,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40390 deadline: 1732378489418, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:13:49,431 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:49,431 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:49,431 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:13:49,431 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:13:49,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40350 deadline: 1732378489419, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:13:49,432 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:49,432 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:49,432 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:13:49,446 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-23T16:13:49,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302. 2024-11-23T16:13:49,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302. as already flushing 2024-11-23T16:13:49,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302. 2024-11-23T16:13:49,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-23T16:13:49,448 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 4550087eb34ce466d194f9ce96539302, NAME => 'TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:13:49,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 4550087eb34ce466d194f9ce96539302, NAME => 'TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:13:49,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45799 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4550087eb34ce466d194f9ce96539302, NAME => 'TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4550087eb34ce466d194f9ce96539302, NAME => 'TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:13:49,580 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:13:49,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40416 deadline: 1732378489578, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:13:49,581 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:49,581 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:13:49,581 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:49,581 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:13:49,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40410 deadline: 1732378489578, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:13:49,582 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:49,582 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:49,582 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:13:49,596 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=121 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp/A/6144aee27cd5447484ee86e1ade2e898 2024-11-23T16:13:49,605 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-23T16:13:49,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302. 2024-11-23T16:13:49,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302. as already flushing 2024-11-23T16:13:49,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302. 2024-11-23T16:13:49,605 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 4550087eb34ce466d194f9ce96539302, NAME => 'TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:13:49,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 4550087eb34ce466d194f9ce96539302, NAME => 'TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:13:49,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45799 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4550087eb34ce466d194f9ce96539302, NAME => 'TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4550087eb34ce466d194f9ce96539302, NAME => 'TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:13:49,615 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp/B/882cb90110d44b59ae01b0192184f7f5 is 50, key is test_row_0/B:col10/1732378429105/Put/seqid=0 2024-11-23T16:13:49,631 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073741856_1032 (size=12001) 2024-11-23T16:13:49,632 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.55 KB at sequenceid=121 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp/B/882cb90110d44b59ae01b0192184f7f5 2024-11-23T16:13:49,646 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:13:49,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40380 deadline: 1732378489638, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:13:49,647 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:49,647 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:49,647 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:13:49,647 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:13:49,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40350 deadline: 1732378489639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:13:49,648 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:13:49,648 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:49,648 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:49,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40390 deadline: 1732378489640, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:13:49,648 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:13:49,649 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:49,649 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:49,649 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:13:49,650 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp/C/e7c83b74b9fb4c8f91f5bb8778cd67c2 is 50, key is test_row_0/C:col10/1732378429105/Put/seqid=0 2024-11-23T16:13:49,668 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073741857_1033 (size=12001) 2024-11-23T16:13:49,672 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=121 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp/C/e7c83b74b9fb4c8f91f5bb8778cd67c2 2024-11-23T16:13:49,686 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp/A/6144aee27cd5447484ee86e1ade2e898 as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/A/6144aee27cd5447484ee86e1ade2e898 2024-11-23T16:13:49,695 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/A/6144aee27cd5447484ee86e1ade2e898, entries=150, sequenceid=121, filesize=11.7 K 2024-11-23T16:13:49,696 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp/B/882cb90110d44b59ae01b0192184f7f5 as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/B/882cb90110d44b59ae01b0192184f7f5 2024-11-23T16:13:49,704 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/B/882cb90110d44b59ae01b0192184f7f5, entries=150, sequenceid=121, filesize=11.7 K 2024-11-23T16:13:49,706 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp/C/e7c83b74b9fb4c8f91f5bb8778cd67c2 as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/C/e7c83b74b9fb4c8f91f5bb8778cd67c2 2024-11-23T16:13:49,715 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/C/e7c83b74b9fb4c8f91f5bb8778cd67c2, entries=150, sequenceid=121, filesize=11.7 K 2024-11-23T16:13:49,717 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~57.75 KB/59135, heapSize ~163.21 KB/167128, currentSize=127.47 KB/130530 for 4550087eb34ce466d194f9ce96539302 in 595ms, sequenceid=121, compaction requested=true 2024-11-23T16:13:49,718 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4550087eb34ce466d194f9ce96539302: 2024-11-23T16:13:49,718 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T16:13:49,720 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4550087eb34ce466d194f9ce96539302:A, priority=-2147483648, current under compaction store size is 1 2024-11-23T16:13:49,720 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38481 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T16:13:49,720 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T16:13:49,720 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HStore(1541): 4550087eb34ce466d194f9ce96539302/A is initiating minor compaction (all files) 2024-11-23T16:13:49,720 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4550087eb34ce466d194f9ce96539302:B, priority=-2147483648, current under compaction store size is 2 2024-11-23T16:13:49,720 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4550087eb34ce466d194f9ce96539302/A in TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302. 2024-11-23T16:13:49,720 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T16:13:49,721 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4550087eb34ce466d194f9ce96539302:C, priority=-2147483648, current under compaction store size is 3 2024-11-23T16:13:49,721 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/A/fc702aca2401483fb69f73fc0cc7511e, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/A/75298ec17fac429488835c77a3eadcc7, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/A/6144aee27cd5447484ee86e1ade2e898] into tmpdir=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp, totalSize=37.6 K 2024-11-23T16:13:49,721 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T16:13:49,721 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T16:13:49,722 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.Compactor(225): Compacting fc702aca2401483fb69f73fc0cc7511e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1732378427360 2024-11-23T16:13:49,722 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.Compactor(225): Compacting 75298ec17fac429488835c77a3eadcc7, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1732378427519 2024-11-23T16:13:49,723 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36141 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T16:13:49,723 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HStore(1541): 4550087eb34ce466d194f9ce96539302/B is initiating minor compaction (all files) 2024-11-23T16:13:49,723 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4550087eb34ce466d194f9ce96539302/B in TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302. 2024-11-23T16:13:49,723 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/B/d356752a25ff4bb58f1ef25e56c4ab3b, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/B/707e1bd9e5814838a99c40af9f3f7468, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/B/882cb90110d44b59ae01b0192184f7f5] into tmpdir=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp, totalSize=35.3 K 2024-11-23T16:13:49,723 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6144aee27cd5447484ee86e1ade2e898, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=121, earliestPutTs=1732378429105 2024-11-23T16:13:49,724 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.Compactor(225): Compacting d356752a25ff4bb58f1ef25e56c4ab3b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1732378427360 2024-11-23T16:13:49,725 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.Compactor(225): Compacting 707e1bd9e5814838a99c40af9f3f7468, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1732378427519 2024-11-23T16:13:49,727 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.Compactor(225): Compacting 882cb90110d44b59ae01b0192184f7f5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=121, earliestPutTs=1732378429105 2024-11-23T16:13:49,751 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4550087eb34ce466d194f9ce96539302#A#compaction#21 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T16:13:49,752 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp/A/81b73bdad0f243829088dad2f30d5500 is 50, key is test_row_0/A:col10/1732378429105/Put/seqid=0 2024-11-23T16:13:49,759 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4550087eb34ce466d194f9ce96539302#B#compaction#22 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T16:13:49,759 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp/B/f6e8502fcd7142d8a289d3ed4d4efc7c is 50, key is test_row_0/B:col10/1732378429105/Put/seqid=0 2024-11-23T16:13:49,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-23T16:13:49,763 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-23T16:13:49,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302. 2024-11-23T16:13:49,763 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2902): Flushing 4550087eb34ce466d194f9ce96539302 3/3 column families, dataSize=127.47 KB heapSize=363.78 KB 2024-11-23T16:13:49,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4550087eb34ce466d194f9ce96539302, store=A 2024-11-23T16:13:49,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-23T16:13:49,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4550087eb34ce466d194f9ce96539302, store=B 2024-11-23T16:13:49,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-23T16:13:49,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4550087eb34ce466d194f9ce96539302, store=C 2024-11-23T16:13:49,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-23T16:13:49,786 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073741858_1034 (size=12241) 2024-11-23T16:13:49,801 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp/A/81b73bdad0f243829088dad2f30d5500 as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/A/81b73bdad0f243829088dad2f30d5500 2024-11-23T16:13:49,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp/A/499bc90d1c6243ad83bb04294c762777 is 50, key is test_row_0/A:col10/1732378429299/Put/seqid=0 2024-11-23T16:13:49,814 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4550087eb34ce466d194f9ce96539302/A of 4550087eb34ce466d194f9ce96539302 into 81b73bdad0f243829088dad2f30d5500(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T16:13:49,814 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4550087eb34ce466d194f9ce96539302: 2024-11-23T16:13:49,815 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., storeName=4550087eb34ce466d194f9ce96539302/A, priority=13, startTime=1732378429718; duration=0sec 2024-11-23T16:13:49,815 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T16:13:49,815 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4550087eb34ce466d194f9ce96539302:A 2024-11-23T16:13:49,815 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T16:13:49,822 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36141 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T16:13:49,822 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HStore(1541): 4550087eb34ce466d194f9ce96539302/C is initiating minor compaction (all files) 2024-11-23T16:13:49,822 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4550087eb34ce466d194f9ce96539302/C in TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302. 2024-11-23T16:13:49,822 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/C/c82cabbae3434cd99b6938c95500884b, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/C/4d8a2528d7474e3585dd0e27ab5d5867, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/C/e7c83b74b9fb4c8f91f5bb8778cd67c2] into tmpdir=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp, totalSize=35.3 K 2024-11-23T16:13:49,823 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.Compactor(225): Compacting c82cabbae3434cd99b6938c95500884b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1732378427360 2024-11-23T16:13:49,824 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4d8a2528d7474e3585dd0e27ab5d5867, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1732378427519 2024-11-23T16:13:49,825 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.Compactor(225): Compacting e7c83b74b9fb4c8f91f5bb8778cd67c2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=121, earliestPutTs=1732378429105 2024-11-23T16:13:49,829 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073741859_1035 (size=12241) 2024-11-23T16:13:49,847 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073741860_1036 (size=12151) 2024-11-23T16:13:49,848 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp/B/f6e8502fcd7142d8a289d3ed4d4efc7c as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/B/f6e8502fcd7142d8a289d3ed4d4efc7c 2024-11-23T16:13:49,848 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=152 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp/A/499bc90d1c6243ad83bb04294c762777 2024-11-23T16:13:49,851 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4550087eb34ce466d194f9ce96539302#C#compaction#24 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T16:13:49,852 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp/C/380abb77007c4eafa3e6f8f635767cd0 is 50, key is test_row_0/C:col10/1732378429105/Put/seqid=0 2024-11-23T16:13:49,869 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4550087eb34ce466d194f9ce96539302/B of 4550087eb34ce466d194f9ce96539302 into f6e8502fcd7142d8a289d3ed4d4efc7c(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T16:13:49,869 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4550087eb34ce466d194f9ce96539302: 2024-11-23T16:13:49,869 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., storeName=4550087eb34ce466d194f9ce96539302/B, priority=13, startTime=1732378429720; duration=0sec 2024-11-23T16:13:49,870 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T16:13:49,870 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4550087eb34ce466d194f9ce96539302:B 2024-11-23T16:13:49,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp/B/99b189b77f60437d8b8abf639d661951 is 50, key is test_row_0/B:col10/1732378429299/Put/seqid=0 2024-11-23T16:13:49,891 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302. as already flushing 2024-11-23T16:13:49,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(8855): Flush requested on 4550087eb34ce466d194f9ce96539302 2024-11-23T16:13:49,904 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073741862_1038 (size=12151) 2024-11-23T16:13:49,905 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=152 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp/B/99b189b77f60437d8b8abf639d661951 2024-11-23T16:13:49,917 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073741861_1037 (size=12241) 2024-11-23T16:13:49,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp/C/2bbcb47f9bc5473b9f50c28c5f1a1409 is 50, key is test_row_0/C:col10/1732378429299/Put/seqid=0 2024-11-23T16:13:49,937 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp/C/380abb77007c4eafa3e6f8f635767cd0 as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/C/380abb77007c4eafa3e6f8f635767cd0 2024-11-23T16:13:49,942 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:13:49,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40416 deadline: 1732378489933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:13:49,943 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:13:49,943 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:49,943 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:49,943 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:13:49,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40410 deadline: 1732378489936, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:13:49,944 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:49,944 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:49,944 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:13:49,949 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4550087eb34ce466d194f9ce96539302/C of 4550087eb34ce466d194f9ce96539302 into 380abb77007c4eafa3e6f8f635767cd0(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T16:13:49,949 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4550087eb34ce466d194f9ce96539302: 2024-11-23T16:13:49,949 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., storeName=4550087eb34ce466d194f9ce96539302/C, priority=13, startTime=1732378429720; duration=0sec 2024-11-23T16:13:49,950 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T16:13:49,950 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4550087eb34ce466d194f9ce96539302:C 2024-11-23T16:13:49,964 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:13:49,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40380 deadline: 1732378489958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:13:49,965 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:49,965 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:13:49,966 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:49,966 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:13:49,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40350 deadline: 1732378489958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:13:49,966 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:49,966 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:49,966 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:13:49,966 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:13:49,967 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073741863_1039 (size=12151) 2024-11-23T16:13:49,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40390 deadline: 1732378489959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:13:49,967 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:49,967 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:49,967 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:13:49,968 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=60.38 KB at sequenceid=152 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp/C/2bbcb47f9bc5473b9f50c28c5f1a1409 2024-11-23T16:13:49,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp/A/499bc90d1c6243ad83bb04294c762777 as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/A/499bc90d1c6243ad83bb04294c762777 2024-11-23T16:13:50,000 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/A/499bc90d1c6243ad83bb04294c762777, entries=150, sequenceid=152, filesize=11.9 K 2024-11-23T16:13:50,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp/B/99b189b77f60437d8b8abf639d661951 as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/B/99b189b77f60437d8b8abf639d661951 2024-11-23T16:13:50,021 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/B/99b189b77f60437d8b8abf639d661951, entries=150, sequenceid=152, filesize=11.9 K 2024-11-23T16:13:50,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp/C/2bbcb47f9bc5473b9f50c28c5f1a1409 as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/C/2bbcb47f9bc5473b9f50c28c5f1a1409 2024-11-23T16:13:50,053 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:13:50,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40410 deadline: 1732378490049, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:13:50,054 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:50,054 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/C/2bbcb47f9bc5473b9f50c28c5f1a1409, entries=150, sequenceid=152, filesize=11.9 K 2024-11-23T16:13:50,054 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:50,054 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:13:50,055 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~363.73 KB/372464, currentSize=53.67 KB/54960 for 4550087eb34ce466d194f9ce96539302 in 292ms, sequenceid=152, compaction requested=false 2024-11-23T16:13:50,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2603): Flush status journal for 4550087eb34ce466d194f9ce96539302: 2024-11-23T16:13:50,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302. 2024-11-23T16:13:50,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=10 2024-11-23T16:13:50,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45799 {}] master.HMaster(4169): Remote procedure done, pid=10 2024-11-23T16:13:50,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(8855): Flush requested on 4550087eb34ce466d194f9ce96539302 2024-11-23T16:13:50,057 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4550087eb34ce466d194f9ce96539302 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-23T16:13:50,058 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4550087eb34ce466d194f9ce96539302, store=A 2024-11-23T16:13:50,058 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-23T16:13:50,058 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4550087eb34ce466d194f9ce96539302, store=B 2024-11-23T16:13:50,058 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-23T16:13:50,058 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4550087eb34ce466d194f9ce96539302, store=C 2024-11-23T16:13:50,058 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-23T16:13:50,063 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=9 2024-11-23T16:13:50,063 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 923 msec 2024-11-23T16:13:50,067 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees in 936 msec 2024-11-23T16:13:50,078 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp/A/24c91d4415f741bca4e2ee0a4646c557 is 50, key is test_row_0/A:col10/1732378430055/Put/seqid=0 2024-11-23T16:13:50,100 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073741864_1040 (size=14541) 2024-11-23T16:13:50,102 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=167 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp/A/24c91d4415f741bca4e2ee0a4646c557 2024-11-23T16:13:50,108 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-23T16:13:50,109 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-23T16:13:50,111 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-23T16:13:50,111 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-23T16:13:50,111 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-23T16:13:50,111 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-23T16:13:50,113 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-23T16:13:50,113 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-23T16:13:50,124 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp/B/9dc39ed6774a49ca89d6ff86db2c56e2 is 50, key is test_row_0/B:col10/1732378430055/Put/seqid=0 2024-11-23T16:13:50,138 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073741865_1041 (size=12151) 2024-11-23T16:13:50,140 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=167 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp/B/9dc39ed6774a49ca89d6ff86db2c56e2 2024-11-23T16:13:50,164 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp/C/35709892451242dcae270339ab4be382 is 50, key is test_row_0/C:col10/1732378430055/Put/seqid=0 2024-11-23T16:13:50,183 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073741866_1042 (size=12151) 2024-11-23T16:13:50,193 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:13:50,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40416 deadline: 1732378490188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:13:50,194 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:50,194 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:50,194 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:13:50,261 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:13:50,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40410 deadline: 1732378490258, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:13:50,262 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:50,262 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:50,262 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:13:50,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-23T16:13:50,269 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-23T16:13:50,269 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45799 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-23T16:13:50,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45799 {}] procedure2.ProcedureExecutor(1139): Stored pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees 2024-11-23T16:13:50,273 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-23T16:13:50,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-23T16:13:50,274 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-23T16:13:50,274 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-23T16:13:50,302 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:13:50,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40416 deadline: 1732378490298, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:13:50,303 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:50,303 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:50,303 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:13:50,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-23T16:13:50,428 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-23T16:13:50,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302. 2024-11-23T16:13:50,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302. as already flushing 2024-11-23T16:13:50,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302. 2024-11-23T16:13:50,428 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 4550087eb34ce466d194f9ce96539302, NAME => 'TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:13:50,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 4550087eb34ce466d194f9ce96539302, NAME => 'TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:13:50,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45799 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4550087eb34ce466d194f9ce96539302, NAME => 'TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4550087eb34ce466d194f9ce96539302, NAME => 'TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:13:50,480 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:13:50,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40380 deadline: 1732378490478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:13:50,481 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:13:50,481 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:50,481 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:50,481 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:13:50,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40350 deadline: 1732378490478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:13:50,482 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:50,482 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:50,482 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:13:50,482 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:13:50,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40390 deadline: 1732378490478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:13:50,482 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:50,482 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:50,483 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:13:50,508 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:13:50,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40416 deadline: 1732378490508, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:13:50,509 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:50,509 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:50,509 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:13:50,573 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:13:50,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40410 deadline: 1732378490568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:13:50,574 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:50,574 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:50,574 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:13:50,582 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-23T16:13:50,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302. 2024-11-23T16:13:50,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302. as already flushing 2024-11-23T16:13:50,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302. 2024-11-23T16:13:50,583 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 4550087eb34ce466d194f9ce96539302, NAME => 'TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:13:50,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 4550087eb34ce466d194f9ce96539302, NAME => 'TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:13:50,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45799 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4550087eb34ce466d194f9ce96539302, NAME => 'TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4550087eb34ce466d194f9ce96539302, NAME => 'TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:13:50,585 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=167 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp/C/35709892451242dcae270339ab4be382 2024-11-23T16:13:50,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-23T16:13:50,598 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp/A/24c91d4415f741bca4e2ee0a4646c557 as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/A/24c91d4415f741bca4e2ee0a4646c557 2024-11-23T16:13:50,608 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/A/24c91d4415f741bca4e2ee0a4646c557, entries=200, sequenceid=167, filesize=14.2 K 2024-11-23T16:13:50,610 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp/B/9dc39ed6774a49ca89d6ff86db2c56e2 as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/B/9dc39ed6774a49ca89d6ff86db2c56e2 2024-11-23T16:13:50,621 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/B/9dc39ed6774a49ca89d6ff86db2c56e2, entries=150, sequenceid=167, filesize=11.9 K 2024-11-23T16:13:50,625 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp/C/35709892451242dcae270339ab4be382 as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/C/35709892451242dcae270339ab4be382 2024-11-23T16:13:50,635 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/C/35709892451242dcae270339ab4be382, entries=150, sequenceid=167, filesize=11.9 K 2024-11-23T16:13:50,637 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 4550087eb34ce466d194f9ce96539302 in 580ms, sequenceid=167, compaction requested=true 2024-11-23T16:13:50,637 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4550087eb34ce466d194f9ce96539302: 2024-11-23T16:13:50,637 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4550087eb34ce466d194f9ce96539302:A, priority=-2147483648, current under compaction store size is 1 2024-11-23T16:13:50,637 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T16:13:50,637 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T16:13:50,637 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4550087eb34ce466d194f9ce96539302:B, priority=-2147483648, current under compaction store size is 2 2024-11-23T16:13:50,637 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T16:13:50,638 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4550087eb34ce466d194f9ce96539302:C, priority=-2147483648, current under compaction store size is 3 2024-11-23T16:13:50,638 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T16:13:50,638 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T16:13:50,640 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38933 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T16:13:50,640 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HStore(1541): 4550087eb34ce466d194f9ce96539302/A is initiating minor compaction (all files) 2024-11-23T16:13:50,640 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4550087eb34ce466d194f9ce96539302/A in TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302. 2024-11-23T16:13:50,640 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/A/81b73bdad0f243829088dad2f30d5500, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/A/499bc90d1c6243ad83bb04294c762777, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/A/24c91d4415f741bca4e2ee0a4646c557] into tmpdir=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp, totalSize=38.0 K 2024-11-23T16:13:50,640 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36543 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T16:13:50,641 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HStore(1541): 4550087eb34ce466d194f9ce96539302/B is initiating minor compaction (all files) 2024-11-23T16:13:50,641 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4550087eb34ce466d194f9ce96539302/B in TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302. 2024-11-23T16:13:50,641 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/B/f6e8502fcd7142d8a289d3ed4d4efc7c, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/B/99b189b77f60437d8b8abf639d661951, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/B/9dc39ed6774a49ca89d6ff86db2c56e2] into tmpdir=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp, totalSize=35.7 K 2024-11-23T16:13:50,641 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.Compactor(225): Compacting 81b73bdad0f243829088dad2f30d5500, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=121, earliestPutTs=1732378429105 2024-11-23T16:13:50,644 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.Compactor(225): Compacting 499bc90d1c6243ad83bb04294c762777, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=152, earliestPutTs=1732378429277 2024-11-23T16:13:50,644 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.Compactor(225): Compacting f6e8502fcd7142d8a289d3ed4d4efc7c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=121, earliestPutTs=1732378429105 2024-11-23T16:13:50,644 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.Compactor(225): Compacting 99b189b77f60437d8b8abf639d661951, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=152, earliestPutTs=1732378429277 2024-11-23T16:13:50,646 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.Compactor(225): Compacting 24c91d4415f741bca4e2ee0a4646c557, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1732378429902 2024-11-23T16:13:50,647 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.Compactor(225): Compacting 9dc39ed6774a49ca89d6ff86db2c56e2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1732378429902 2024-11-23T16:13:50,671 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4550087eb34ce466d194f9ce96539302#A#compaction#30 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T16:13:50,672 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp/A/4af5583f912e4fceae6174b1df9bc52c is 50, key is test_row_0/A:col10/1732378430055/Put/seqid=0 2024-11-23T16:13:50,675 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4550087eb34ce466d194f9ce96539302#B#compaction#31 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T16:13:50,676 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp/B/fb82bfda637d4937a56d7ceecd6d635d is 50, key is test_row_0/B:col10/1732378430055/Put/seqid=0 2024-11-23T16:13:50,699 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073741867_1043 (size=12493) 2024-11-23T16:13:50,700 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073741868_1044 (size=12493) 2024-11-23T16:13:50,718 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp/B/fb82bfda637d4937a56d7ceecd6d635d as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/B/fb82bfda637d4937a56d7ceecd6d635d 2024-11-23T16:13:50,728 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp/A/4af5583f912e4fceae6174b1df9bc52c as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/A/4af5583f912e4fceae6174b1df9bc52c 2024-11-23T16:13:50,737 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-23T16:13:50,737 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4550087eb34ce466d194f9ce96539302/B of 4550087eb34ce466d194f9ce96539302 into fb82bfda637d4937a56d7ceecd6d635d(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T16:13:50,738 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4550087eb34ce466d194f9ce96539302: 2024-11-23T16:13:50,738 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., storeName=4550087eb34ce466d194f9ce96539302/B, priority=13, startTime=1732378430637; duration=0sec 2024-11-23T16:13:50,738 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T16:13:50,738 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4550087eb34ce466d194f9ce96539302:B 2024-11-23T16:13:50,738 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T16:13:50,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302. 2024-11-23T16:13:50,739 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2902): Flushing 4550087eb34ce466d194f9ce96539302 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-23T16:13:50,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4550087eb34ce466d194f9ce96539302, store=A 2024-11-23T16:13:50,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-23T16:13:50,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4550087eb34ce466d194f9ce96539302, store=B 2024-11-23T16:13:50,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-23T16:13:50,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4550087eb34ce466d194f9ce96539302, store=C 2024-11-23T16:13:50,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-23T16:13:50,742 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36543 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T16:13:50,743 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HStore(1541): 4550087eb34ce466d194f9ce96539302/C is initiating minor compaction (all files) 2024-11-23T16:13:50,744 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4550087eb34ce466d194f9ce96539302/C in TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302. 2024-11-23T16:13:50,744 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4550087eb34ce466d194f9ce96539302/A of 4550087eb34ce466d194f9ce96539302 into 4af5583f912e4fceae6174b1df9bc52c(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T16:13:50,744 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4550087eb34ce466d194f9ce96539302: 2024-11-23T16:13:50,744 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., storeName=4550087eb34ce466d194f9ce96539302/A, priority=13, startTime=1732378430637; duration=0sec 2024-11-23T16:13:50,745 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T16:13:50,745 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4550087eb34ce466d194f9ce96539302:A 2024-11-23T16:13:50,745 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/C/380abb77007c4eafa3e6f8f635767cd0, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/C/2bbcb47f9bc5473b9f50c28c5f1a1409, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/C/35709892451242dcae270339ab4be382] into tmpdir=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp, totalSize=35.7 K 2024-11-23T16:13:50,747 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.Compactor(225): Compacting 380abb77007c4eafa3e6f8f635767cd0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=121, earliestPutTs=1732378429105 2024-11-23T16:13:50,747 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.Compactor(225): Compacting 2bbcb47f9bc5473b9f50c28c5f1a1409, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=152, earliestPutTs=1732378429277 2024-11-23T16:13:50,757 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.Compactor(225): Compacting 35709892451242dcae270339ab4be382, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1732378429902 2024-11-23T16:13:50,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp/A/6928613c49f8437aa7f059a0eff46d3d is 50, key is test_row_0/A:col10/1732378430166/Put/seqid=0 2024-11-23T16:13:50,777 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073741869_1045 (size=12151) 2024-11-23T16:13:50,783 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4550087eb34ce466d194f9ce96539302#C#compaction#33 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T16:13:50,784 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp/C/6b7d6b2c297a44d69495b87450b88e76 is 50, key is test_row_0/C:col10/1732378430055/Put/seqid=0 2024-11-23T16:13:50,817 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073741870_1046 (size=12493) 2024-11-23T16:13:50,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(8855): Flush requested on 4550087eb34ce466d194f9ce96539302 2024-11-23T16:13:50,820 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302. as already flushing 2024-11-23T16:13:50,868 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:13:50,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40416 deadline: 1732378490866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:13:50,869 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:50,869 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:50,869 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:13:50,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-23T16:13:50,985 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:13:50,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40416 deadline: 1732378490978, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:13:50,986 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:50,986 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:50,986 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:13:51,078 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:13:51,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40410 deadline: 1732378491078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:13:51,079 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:51,080 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:51,080 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:13:51,179 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=188 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp/A/6928613c49f8437aa7f059a0eff46d3d 2024-11-23T16:13:51,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp/B/7c46d8a4b3b74a68befd5b9212928de6 is 50, key is test_row_0/B:col10/1732378430166/Put/seqid=0 2024-11-23T16:13:51,202 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:13:51,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40416 deadline: 1732378491198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:13:51,203 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:51,203 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:51,204 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:13:51,215 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073741871_1047 (size=12151) 2024-11-23T16:13:51,220 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=188 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp/B/7c46d8a4b3b74a68befd5b9212928de6 2024-11-23T16:13:51,226 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp/C/6b7d6b2c297a44d69495b87450b88e76 as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/C/6b7d6b2c297a44d69495b87450b88e76 2024-11-23T16:13:51,240 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4550087eb34ce466d194f9ce96539302/C of 4550087eb34ce466d194f9ce96539302 into 6b7d6b2c297a44d69495b87450b88e76(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T16:13:51,241 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4550087eb34ce466d194f9ce96539302: 2024-11-23T16:13:51,241 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., storeName=4550087eb34ce466d194f9ce96539302/C, priority=13, startTime=1732378430637; duration=0sec 2024-11-23T16:13:51,241 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T16:13:51,241 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4550087eb34ce466d194f9ce96539302:C 2024-11-23T16:13:51,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp/C/a0d5a3ced13040c89b6187089f1c5f4e is 50, key is test_row_0/C:col10/1732378430166/Put/seqid=0 2024-11-23T16:13:51,277 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073741872_1048 (size=12151) 2024-11-23T16:13:51,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-23T16:13:51,488 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:13:51,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40380 deadline: 1732378491488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:13:51,489 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:13:51,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40350 deadline: 1732378491488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:13:51,490 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:13:51,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40390 deadline: 1732378491488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:13:51,493 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 1a7c5470b472,44715,1732378420375 for 'test_row_1' in 4550087eb34ce466d194f9ce96539302 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2188 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-23T16:13:51,493 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 1a7c5470b472,44715,1732378420375 for 'test_row_2' in 4550087eb34ce466d194f9ce96539302 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2188 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-23T16:13:51,494 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:51,494 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:51,494 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:51,494 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:13:51,494 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:51,494 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:13:51,495 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 1a7c5470b472,44715,1732378420375 for 'test_row_0' in 4550087eb34ce466d194f9ce96539302 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2188 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-23T16:13:51,495 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:51,495 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:51,495 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:13:51,508 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:13:51,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40416 deadline: 1732378491508, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:13:51,509 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:51,509 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:51,509 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:13:51,679 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=188 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp/C/a0d5a3ced13040c89b6187089f1c5f4e 2024-11-23T16:13:51,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp/A/6928613c49f8437aa7f059a0eff46d3d as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/A/6928613c49f8437aa7f059a0eff46d3d 2024-11-23T16:13:51,702 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/A/6928613c49f8437aa7f059a0eff46d3d, entries=150, sequenceid=188, filesize=11.9 K 2024-11-23T16:13:51,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp/B/7c46d8a4b3b74a68befd5b9212928de6 as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/B/7c46d8a4b3b74a68befd5b9212928de6 2024-11-23T16:13:51,715 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/B/7c46d8a4b3b74a68befd5b9212928de6, entries=150, sequenceid=188, filesize=11.9 K 2024-11-23T16:13:51,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp/C/a0d5a3ced13040c89b6187089f1c5f4e as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/C/a0d5a3ced13040c89b6187089f1c5f4e 2024-11-23T16:13:51,726 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/C/a0d5a3ced13040c89b6187089f1c5f4e, entries=150, sequenceid=188, filesize=11.9 K 2024-11-23T16:13:51,732 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 4550087eb34ce466d194f9ce96539302 in 994ms, sequenceid=188, compaction requested=false 2024-11-23T16:13:51,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2603): Flush status journal for 4550087eb34ce466d194f9ce96539302: 2024-11-23T16:13:51,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302. 2024-11-23T16:13:51,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=12 2024-11-23T16:13:51,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45799 {}] master.HMaster(4169): Remote procedure done, pid=12 2024-11-23T16:13:51,741 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-11-23T16:13:51,741 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4630 sec 2024-11-23T16:13:51,744 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees in 1.4720 sec 2024-11-23T16:13:52,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(8855): Flush requested on 4550087eb34ce466d194f9ce96539302 2024-11-23T16:13:52,025 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4550087eb34ce466d194f9ce96539302 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-23T16:13:52,026 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4550087eb34ce466d194f9ce96539302, store=A 2024-11-23T16:13:52,026 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-23T16:13:52,026 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4550087eb34ce466d194f9ce96539302, store=B 2024-11-23T16:13:52,026 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-23T16:13:52,026 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4550087eb34ce466d194f9ce96539302, store=C 2024-11-23T16:13:52,026 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-23T16:13:52,049 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp/A/a19ce2b875ff463bae8bf01b447e2b3d is 50, key is test_row_0/A:col10/1732378432023/Put/seqid=0 2024-11-23T16:13:52,071 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073741873_1049 (size=12151) 2024-11-23T16:13:52,073 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=203 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp/A/a19ce2b875ff463bae8bf01b447e2b3d 2024-11-23T16:13:52,085 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp/B/02e054afe23a43ba83be6f0c5a06abbe is 50, key is test_row_0/B:col10/1732378432023/Put/seqid=0 2024-11-23T16:13:52,094 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073741874_1050 (size=12151) 2024-11-23T16:13:52,101 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:13:52,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40410 deadline: 1732378492098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:13:52,102 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:52,102 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:52,102 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:13:52,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40416 deadline: 1732378492101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:13:52,103 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:52,103 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:52,103 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:13:52,103 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:13:52,210 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:13:52,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40416 deadline: 1732378492208, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:13:52,211 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:13:52,212 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:52,212 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:52,212 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:13:52,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40410 deadline: 1732378492208, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:13:52,213 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:52,213 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:52,213 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:13:52,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-23T16:13:52,418 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:13:52,419 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-23T16:13:52,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40416 deadline: 1732378492418, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:13:52,419 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45799 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-23T16:13:52,419 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:52,419 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:13:52,420 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:52,420 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:13:52,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40410 deadline: 1732378492418, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:13:52,420 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:52,420 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:52,420 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:13:52,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45799 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees 2024-11-23T16:13:52,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-23T16:13:52,422 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-23T16:13:52,423 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-23T16:13:52,424 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-23T16:13:52,495 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=203 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp/B/02e054afe23a43ba83be6f0c5a06abbe 2024-11-23T16:13:52,518 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp/C/1646c45305c240aebf7c4ed037f78e08 is 50, key is test_row_0/C:col10/1732378432023/Put/seqid=0 2024-11-23T16:13:52,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-23T16:13:52,546 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073741875_1051 (size=12151) 2024-11-23T16:13:52,548 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=203 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp/C/1646c45305c240aebf7c4ed037f78e08 2024-11-23T16:13:52,561 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp/A/a19ce2b875ff463bae8bf01b447e2b3d as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/A/a19ce2b875ff463bae8bf01b447e2b3d 2024-11-23T16:13:52,569 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/A/a19ce2b875ff463bae8bf01b447e2b3d, entries=150, sequenceid=203, filesize=11.9 K 2024-11-23T16:13:52,571 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp/B/02e054afe23a43ba83be6f0c5a06abbe as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/B/02e054afe23a43ba83be6f0c5a06abbe 2024-11-23T16:13:52,577 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-23T16:13:52,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302. 2024-11-23T16:13:52,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302. as already flushing 2024-11-23T16:13:52,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302. 2024-11-23T16:13:52,578 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 4550087eb34ce466d194f9ce96539302, NAME => 'TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:13:52,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 4550087eb34ce466d194f9ce96539302, NAME => 'TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:13:52,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45799 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4550087eb34ce466d194f9ce96539302, NAME => 'TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4550087eb34ce466d194f9ce96539302, NAME => 'TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:13:52,592 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/B/02e054afe23a43ba83be6f0c5a06abbe, entries=150, sequenceid=203, filesize=11.9 K 2024-11-23T16:13:52,593 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp/C/1646c45305c240aebf7c4ed037f78e08 as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/C/1646c45305c240aebf7c4ed037f78e08 2024-11-23T16:13:52,600 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/C/1646c45305c240aebf7c4ed037f78e08, entries=150, sequenceid=203, filesize=11.9 K 2024-11-23T16:13:52,611 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 4550087eb34ce466d194f9ce96539302 in 586ms, sequenceid=203, compaction requested=true 2024-11-23T16:13:52,612 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4550087eb34ce466d194f9ce96539302: 2024-11-23T16:13:52,612 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4550087eb34ce466d194f9ce96539302:A, priority=-2147483648, current under compaction store size is 1 2024-11-23T16:13:52,612 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T16:13:52,612 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T16:13:52,612 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4550087eb34ce466d194f9ce96539302:B, priority=-2147483648, current under compaction store size is 2 2024-11-23T16:13:52,612 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T16:13:52,612 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4550087eb34ce466d194f9ce96539302:C, priority=-2147483648, current under compaction store size is 3 2024-11-23T16:13:52,612 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-23T16:13:52,612 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T16:13:52,617 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T16:13:52,617 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HStore(1541): 4550087eb34ce466d194f9ce96539302/A is initiating minor compaction (all files) 2024-11-23T16:13:52,617 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T16:13:52,617 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4550087eb34ce466d194f9ce96539302/A in TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302. 2024-11-23T16:13:52,617 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HStore(1541): 4550087eb34ce466d194f9ce96539302/B is initiating minor compaction (all files) 2024-11-23T16:13:52,617 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/A/4af5583f912e4fceae6174b1df9bc52c, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/A/6928613c49f8437aa7f059a0eff46d3d, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/A/a19ce2b875ff463bae8bf01b447e2b3d] into tmpdir=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp, totalSize=35.9 K 2024-11-23T16:13:52,617 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4550087eb34ce466d194f9ce96539302/B in TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302. 2024-11-23T16:13:52,617 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/B/fb82bfda637d4937a56d7ceecd6d635d, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/B/7c46d8a4b3b74a68befd5b9212928de6, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/B/02e054afe23a43ba83be6f0c5a06abbe] into tmpdir=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp, totalSize=35.9 K 2024-11-23T16:13:52,618 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4af5583f912e4fceae6174b1df9bc52c, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1732378429902 2024-11-23T16:13:52,618 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.Compactor(225): Compacting fb82bfda637d4937a56d7ceecd6d635d, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1732378429902 2024-11-23T16:13:52,619 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6928613c49f8437aa7f059a0eff46d3d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=188, earliestPutTs=1732378430166 2024-11-23T16:13:52,619 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.Compactor(225): Compacting 7c46d8a4b3b74a68befd5b9212928de6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=188, earliestPutTs=1732378430166 2024-11-23T16:13:52,620 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.Compactor(225): Compacting a19ce2b875ff463bae8bf01b447e2b3d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=203, earliestPutTs=1732378430847 2024-11-23T16:13:52,620 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.Compactor(225): Compacting 02e054afe23a43ba83be6f0c5a06abbe, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=203, earliestPutTs=1732378430847 2024-11-23T16:13:52,642 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4550087eb34ce466d194f9ce96539302#A#compaction#39 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-23T16:13:52,643 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4550087eb34ce466d194f9ce96539302#B#compaction#40 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T16:13:52,643 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp/A/6caa843f43804b5b838656e781bfd55a is 50, key is test_row_0/A:col10/1732378432023/Put/seqid=0 2024-11-23T16:13:52,643 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp/B/92d606bbe87a4d2ca12c45d4f9e9bd80 is 50, key is test_row_0/B:col10/1732378432023/Put/seqid=0 2024-11-23T16:13:52,679 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073741876_1052 (size=12595) 2024-11-23T16:13:52,688 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073741877_1053 (size=12595) 2024-11-23T16:13:52,700 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp/A/6caa843f43804b5b838656e781bfd55a as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/A/6caa843f43804b5b838656e781bfd55a 2024-11-23T16:13:52,709 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4550087eb34ce466d194f9ce96539302/A of 4550087eb34ce466d194f9ce96539302 into 6caa843f43804b5b838656e781bfd55a(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T16:13:52,710 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4550087eb34ce466d194f9ce96539302: 2024-11-23T16:13:52,710 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., storeName=4550087eb34ce466d194f9ce96539302/A, priority=13, startTime=1732378432612; duration=0sec 2024-11-23T16:13:52,710 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T16:13:52,710 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4550087eb34ce466d194f9ce96539302:A 2024-11-23T16:13:52,710 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T16:13:52,716 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T16:13:52,716 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HStore(1541): 4550087eb34ce466d194f9ce96539302/C is initiating minor compaction (all files) 2024-11-23T16:13:52,716 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4550087eb34ce466d194f9ce96539302/C in TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302. 2024-11-23T16:13:52,716 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/C/6b7d6b2c297a44d69495b87450b88e76, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/C/a0d5a3ced13040c89b6187089f1c5f4e, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/C/1646c45305c240aebf7c4ed037f78e08] into tmpdir=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp, totalSize=35.9 K 2024-11-23T16:13:52,717 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6b7d6b2c297a44d69495b87450b88e76, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1732378429902 2024-11-23T16:13:52,720 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.Compactor(225): Compacting a0d5a3ced13040c89b6187089f1c5f4e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=188, earliestPutTs=1732378430166 2024-11-23T16:13:52,724 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1646c45305c240aebf7c4ed037f78e08, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=203, earliestPutTs=1732378430847 2024-11-23T16:13:52,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(8855): Flush requested on 4550087eb34ce466d194f9ce96539302 2024-11-23T16:13:52,730 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4550087eb34ce466d194f9ce96539302 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-23T16:13:52,732 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4550087eb34ce466d194f9ce96539302, store=A 2024-11-23T16:13:52,732 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-23T16:13:52,732 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-23T16:13:52,732 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4550087eb34ce466d194f9ce96539302, store=B 2024-11-23T16:13:52,732 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-23T16:13:52,733 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4550087eb34ce466d194f9ce96539302, store=C 2024-11-23T16:13:52,733 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-23T16:13:52,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302. 2024-11-23T16:13:52,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302. as already flushing 2024-11-23T16:13:52,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302. 2024-11-23T16:13:52,733 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 4550087eb34ce466d194f9ce96539302, NAME => 'TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:13:52,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 4550087eb34ce466d194f9ce96539302, NAME => 'TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:13:52,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45799 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4550087eb34ce466d194f9ce96539302, NAME => 'TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4550087eb34ce466d194f9ce96539302, NAME => 'TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:13:52,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-23T16:13:52,753 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp/A/4f136979e1eb43d1a21d5f90e040a57a is 50, key is test_row_0/A:col10/1732378432099/Put/seqid=0 2024-11-23T16:13:52,766 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073741878_1054 (size=16931) 2024-11-23T16:13:52,767 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=225 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp/A/4f136979e1eb43d1a21d5f90e040a57a 2024-11-23T16:13:52,771 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:13:52,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40416 deadline: 1732378492766, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:13:52,772 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:52,772 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:52,772 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:13:52,772 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:13:52,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40410 deadline: 1732378492770, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:13:52,773 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:52,773 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:52,773 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:13:52,774 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4550087eb34ce466d194f9ce96539302#C#compaction#42 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T16:13:52,775 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp/C/37dc0b6979544dcb873b925da413dc17 is 50, key is test_row_0/C:col10/1732378432023/Put/seqid=0 2024-11-23T16:13:52,787 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp/B/4bb8355660b944d6ad482d53cb9d269f is 50, key is test_row_0/B:col10/1732378432099/Put/seqid=0 2024-11-23T16:13:52,811 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073741879_1055 (size=12595) 2024-11-23T16:13:52,819 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073741880_1056 (size=12151) 2024-11-23T16:13:52,820 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=225 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp/B/4bb8355660b944d6ad482d53cb9d269f 2024-11-23T16:13:52,828 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp/C/37dc0b6979544dcb873b925da413dc17 as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/C/37dc0b6979544dcb873b925da413dc17 2024-11-23T16:13:52,841 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp/C/22c4912c178e436d89d44351fd1301d5 is 50, key is test_row_0/C:col10/1732378432099/Put/seqid=0 2024-11-23T16:13:52,843 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4550087eb34ce466d194f9ce96539302/C of 4550087eb34ce466d194f9ce96539302 into 37dc0b6979544dcb873b925da413dc17(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T16:13:52,843 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4550087eb34ce466d194f9ce96539302: 2024-11-23T16:13:52,843 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., storeName=4550087eb34ce466d194f9ce96539302/C, priority=13, startTime=1732378432612; duration=0sec 2024-11-23T16:13:52,843 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T16:13:52,843 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4550087eb34ce466d194f9ce96539302:C 2024-11-23T16:13:52,870 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073741881_1057 (size=12151) 2024-11-23T16:13:52,882 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:13:52,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40410 deadline: 1732378492878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:13:52,885 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:13:52,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40416 deadline: 1732378492878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:13:52,885 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:52,886 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:52,886 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:13:52,886 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:52,886 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:52,886 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:13:52,888 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-23T16:13:52,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302. 2024-11-23T16:13:52,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302. as already flushing 2024-11-23T16:13:52,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302. 2024-11-23T16:13:52,889 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 4550087eb34ce466d194f9ce96539302, NAME => 'TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:13:52,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 4550087eb34ce466d194f9ce96539302, NAME => 'TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:13:52,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45799 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4550087eb34ce466d194f9ce96539302, NAME => 'TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4550087eb34ce466d194f9ce96539302, NAME => 'TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:13:53,044 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-23T16:13:53,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302. 2024-11-23T16:13:53,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302. as already flushing 2024-11-23T16:13:53,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302. 2024-11-23T16:13:53,044 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 4550087eb34ce466d194f9ce96539302, NAME => 'TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:13:53,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 4550087eb34ce466d194f9ce96539302, NAME => 'TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:13:53,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45799 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4550087eb34ce466d194f9ce96539302, NAME => 'TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4550087eb34ce466d194f9ce96539302, NAME => 'TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:13:53,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-23T16:13:53,089 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:13:53,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40410 deadline: 1732378493089, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:13:53,094 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:53,094 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:53,094 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:13:53,094 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:13:53,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40416 deadline: 1732378493091, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:13:53,096 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:53,096 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:53,096 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:13:53,103 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp/B/92d606bbe87a4d2ca12c45d4f9e9bd80 as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/B/92d606bbe87a4d2ca12c45d4f9e9bd80 2024-11-23T16:13:53,118 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4550087eb34ce466d194f9ce96539302/B of 4550087eb34ce466d194f9ce96539302 into 92d606bbe87a4d2ca12c45d4f9e9bd80(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T16:13:53,118 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4550087eb34ce466d194f9ce96539302: 2024-11-23T16:13:53,118 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., storeName=4550087eb34ce466d194f9ce96539302/B, priority=13, startTime=1732378432612; duration=0sec 2024-11-23T16:13:53,118 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T16:13:53,119 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4550087eb34ce466d194f9ce96539302:B 2024-11-23T16:13:53,198 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-23T16:13:53,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302. 2024-11-23T16:13:53,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302. as already flushing 2024-11-23T16:13:53,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302. 2024-11-23T16:13:53,199 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 4550087eb34ce466d194f9ce96539302, NAME => 'TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:13:53,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 4550087eb34ce466d194f9ce96539302, NAME => 'TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:13:53,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45799 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4550087eb34ce466d194f9ce96539302, NAME => 'TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4550087eb34ce466d194f9ce96539302, NAME => 'TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:13:53,270 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=225 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp/C/22c4912c178e436d89d44351fd1301d5 2024-11-23T16:13:53,278 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp/A/4f136979e1eb43d1a21d5f90e040a57a as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/A/4f136979e1eb43d1a21d5f90e040a57a 2024-11-23T16:13:53,284 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/A/4f136979e1eb43d1a21d5f90e040a57a, entries=250, sequenceid=225, filesize=16.5 K 2024-11-23T16:13:53,302 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp/B/4bb8355660b944d6ad482d53cb9d269f as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/B/4bb8355660b944d6ad482d53cb9d269f 2024-11-23T16:13:53,309 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/B/4bb8355660b944d6ad482d53cb9d269f, entries=150, sequenceid=225, filesize=11.9 K 2024-11-23T16:13:53,311 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp/C/22c4912c178e436d89d44351fd1301d5 as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/C/22c4912c178e436d89d44351fd1301d5 2024-11-23T16:13:53,317 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/C/22c4912c178e436d89d44351fd1301d5, entries=150, sequenceid=225, filesize=11.9 K 2024-11-23T16:13:53,319 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=53.67 KB/54960 for 4550087eb34ce466d194f9ce96539302 in 588ms, sequenceid=225, compaction requested=false 2024-11-23T16:13:53,319 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4550087eb34ce466d194f9ce96539302: 2024-11-23T16:13:53,353 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-23T16:13:53,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302. 2024-11-23T16:13:53,354 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2902): Flushing 4550087eb34ce466d194f9ce96539302 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-23T16:13:53,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4550087eb34ce466d194f9ce96539302, store=A 2024-11-23T16:13:53,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-23T16:13:53,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4550087eb34ce466d194f9ce96539302, store=B 2024-11-23T16:13:53,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-23T16:13:53,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4550087eb34ce466d194f9ce96539302, store=C 2024-11-23T16:13:53,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-23T16:13:53,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp/A/7f44ac7f49c542b1bf09ac3260d5a75c is 50, key is test_row_0/A:col10/1732378432763/Put/seqid=0 2024-11-23T16:13:53,375 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073741882_1058 (size=12151) 2024-11-23T16:13:53,382 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=238 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp/A/7f44ac7f49c542b1bf09ac3260d5a75c 2024-11-23T16:13:53,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp/B/2f5bc73e426a4ab4835bb733a76492e9 is 50, key is test_row_0/B:col10/1732378432763/Put/seqid=0 2024-11-23T16:13:53,398 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073741883_1059 (size=12151) 2024-11-23T16:13:53,400 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=238 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp/B/2f5bc73e426a4ab4835bb733a76492e9 2024-11-23T16:13:53,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(8855): Flush requested on 4550087eb34ce466d194f9ce96539302 2024-11-23T16:13:53,400 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302. as already flushing 2024-11-23T16:13:53,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp/C/9d94f4df87b142cd9d4b3fc9571fe4d0 is 50, key is test_row_0/C:col10/1732378432763/Put/seqid=0 2024-11-23T16:13:53,427 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073741884_1060 (size=12151) 2024-11-23T16:13:53,467 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1a7c5470b472:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732378433433, max timestamp=1732378433463Num uniques 859; 2024-11-23T16:13:53,467 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1a7c5470b472:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-23T16:13:53,467 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1a7c5470b472:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732378433433, max timestamp=1732378433463Num uniques 859; 2024-11-23T16:13:53,467 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1a7c5470b472:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732378433433, max timestamp=1732378433463Num uniques 859; 2024-11-23T16:13:53,467 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1a7c5470b472:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-23T16:13:53,468 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1a7c5470b472:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-23T16:13:53,506 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:13:53,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40416 deadline: 1732378493499, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:13:53,507 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:53,508 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:53,508 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:13:53,508 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:13:53,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40410 deadline: 1732378493503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:13:53,509 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:53,509 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:53,509 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:13:53,512 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:13:53,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40350 deadline: 1732378493508, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:13:53,513 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:13:53,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40380 deadline: 1732378493508, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:13:53,513 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 1a7c5470b472,44715,1732378420375 for 'test_row_2' in 4550087eb34ce466d194f9ce96539302 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4211 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-23T16:13:53,514 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:53,514 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:13:53,514 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:53,514 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:13:53,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40390 deadline: 1732378493508, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:13:53,514 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 1a7c5470b472,44715,1732378420375 for 'test_row_0' in 4550087eb34ce466d194f9ce96539302 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4213 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-23T16:13:53,515 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:53,515 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:53,515 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:13:53,515 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 1a7c5470b472,44715,1732378420375 for 'test_row_1' in 4550087eb34ce466d194f9ce96539302 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4212 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-23T16:13:53,515 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:53,516 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:53,516 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:13:53,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-23T16:13:53,621 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:13:53,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40416 deadline: 1732378493618, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:13:53,622 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:53,622 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:13:53,622 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:53,622 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:13:53,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40410 deadline: 1732378493619, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:13:53,622 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:53,622 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:53,622 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:13:53,828 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=238 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp/C/9d94f4df87b142cd9d4b3fc9571fe4d0 2024-11-23T16:13:53,831 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:13:53,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40416 deadline: 1732378493829, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:13:53,832 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:53,832 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:53,832 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:13:53,832 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:13:53,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40410 deadline: 1732378493829, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:13:53,833 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:53,833 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:53,833 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:13:53,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp/A/7f44ac7f49c542b1bf09ac3260d5a75c as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/A/7f44ac7f49c542b1bf09ac3260d5a75c 2024-11-23T16:13:53,848 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/A/7f44ac7f49c542b1bf09ac3260d5a75c, entries=150, sequenceid=238, filesize=11.9 K 2024-11-23T16:13:53,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp/B/2f5bc73e426a4ab4835bb733a76492e9 as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/B/2f5bc73e426a4ab4835bb733a76492e9 2024-11-23T16:13:53,859 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/B/2f5bc73e426a4ab4835bb733a76492e9, entries=150, sequenceid=238, filesize=11.9 K 2024-11-23T16:13:53,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp/C/9d94f4df87b142cd9d4b3fc9571fe4d0 as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/C/9d94f4df87b142cd9d4b3fc9571fe4d0 2024-11-23T16:13:53,869 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/C/9d94f4df87b142cd9d4b3fc9571fe4d0, entries=150, sequenceid=238, filesize=11.9 K 2024-11-23T16:13:53,870 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 4550087eb34ce466d194f9ce96539302 in 516ms, sequenceid=238, compaction requested=true 2024-11-23T16:13:53,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2603): Flush status journal for 4550087eb34ce466d194f9ce96539302: 2024-11-23T16:13:53,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302. 2024-11-23T16:13:53,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=14 2024-11-23T16:13:53,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45799 {}] master.HMaster(4169): Remote procedure done, pid=14 2024-11-23T16:13:53,876 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-11-23T16:13:53,876 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4490 sec 2024-11-23T16:13:53,879 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees in 1.4580 sec 2024-11-23T16:13:54,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(8855): Flush requested on 4550087eb34ce466d194f9ce96539302 2024-11-23T16:13:54,143 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4550087eb34ce466d194f9ce96539302 3/3 column families, dataSize=134.18 KB heapSize=375.61 KB 2024-11-23T16:13:54,143 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4550087eb34ce466d194f9ce96539302, store=A 2024-11-23T16:13:54,143 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-23T16:13:54,143 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4550087eb34ce466d194f9ce96539302, store=B 2024-11-23T16:13:54,144 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-23T16:13:54,144 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4550087eb34ce466d194f9ce96539302, store=C 2024-11-23T16:13:54,144 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-23T16:13:54,153 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp/A/7b655553f25d453683d972d6ddb07d66 is 50, key is test_row_0/A:col10/1732378434141/Put/seqid=0 2024-11-23T16:13:54,159 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073741885_1061 (size=14741) 2024-11-23T16:13:54,187 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:13:54,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40416 deadline: 1732378494185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:13:54,188 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:54,188 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:54,189 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:13:54,190 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:13:54,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40410 deadline: 1732378494188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:13:54,191 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:54,191 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:54,191 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:13:54,300 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:13:54,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40416 deadline: 1732378494298, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:13:54,301 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:54,301 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:13:54,301 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:54,301 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:13:54,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40410 deadline: 1732378494299, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:13:54,301 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:54,301 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:54,301 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:13:54,512 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:13:54,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40416 deadline: 1732378494508, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:13:54,513 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:54,513 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:54,513 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:13:54,513 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:13:54,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40410 deadline: 1732378494508, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:13:54,514 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:54,514 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:54,514 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:13:54,560 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=273 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp/A/7b655553f25d453683d972d6ddb07d66 2024-11-23T16:13:54,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-23T16:13:54,569 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-23T16:13:54,569 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45799 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-23T16:13:54,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45799 {}] procedure2.ProcedureExecutor(1139): Stored pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees 2024-11-23T16:13:54,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-23T16:13:54,573 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-23T16:13:54,574 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-23T16:13:54,574 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=15, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-23T16:13:54,580 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp/B/237cc2230d8347239f11d37f868210a8 is 50, key is test_row_0/B:col10/1732378434141/Put/seqid=0 2024-11-23T16:13:54,585 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073741886_1062 (size=12301) 2024-11-23T16:13:54,586 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=273 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp/B/237cc2230d8347239f11d37f868210a8 2024-11-23T16:13:54,596 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp/C/b9fd72e4d8d742839a4e929b4e5943d3 is 50, key is test_row_0/C:col10/1732378434141/Put/seqid=0 2024-11-23T16:13:54,604 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073741887_1063 (size=12301) 2024-11-23T16:13:54,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-23T16:13:54,727 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-23T16:13:54,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302. 2024-11-23T16:13:54,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302. as already flushing 2024-11-23T16:13:54,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302. 2024-11-23T16:13:54,727 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 4550087eb34ce466d194f9ce96539302, NAME => 'TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:13:54,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 4550087eb34ce466d194f9ce96539302, NAME => 'TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:13:54,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45799 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4550087eb34ce466d194f9ce96539302, NAME => 'TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4550087eb34ce466d194f9ce96539302, NAME => 'TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:13:54,825 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:13:54,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40416 deadline: 1732378494818, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:13:54,826 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:54,826 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:54,826 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:13:54,826 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:13:54,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40410 deadline: 1732378494818, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:13:54,826 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:54,827 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:54,827 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:13:54,881 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-23T16:13:54,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302. 2024-11-23T16:13:54,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302. as already flushing 2024-11-23T16:13:54,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302. 2024-11-23T16:13:54,882 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 4550087eb34ce466d194f9ce96539302, NAME => 'TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:13:54,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 4550087eb34ce466d194f9ce96539302, NAME => 'TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:13:54,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45799 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4550087eb34ce466d194f9ce96539302, NAME => 'TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4550087eb34ce466d194f9ce96539302, NAME => 'TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:13:54,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-23T16:13:55,005 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=273 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp/C/b9fd72e4d8d742839a4e929b4e5943d3 2024-11-23T16:13:55,014 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp/A/7b655553f25d453683d972d6ddb07d66 as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/A/7b655553f25d453683d972d6ddb07d66 2024-11-23T16:13:55,020 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/A/7b655553f25d453683d972d6ddb07d66, entries=200, sequenceid=273, filesize=14.4 K 2024-11-23T16:13:55,021 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp/B/237cc2230d8347239f11d37f868210a8 as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/B/237cc2230d8347239f11d37f868210a8 2024-11-23T16:13:55,028 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/B/237cc2230d8347239f11d37f868210a8, entries=150, sequenceid=273, filesize=12.0 K 2024-11-23T16:13:55,029 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp/C/b9fd72e4d8d742839a4e929b4e5943d3 as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/C/b9fd72e4d8d742839a4e929b4e5943d3 2024-11-23T16:13:55,035 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/C/b9fd72e4d8d742839a4e929b4e5943d3, entries=150, sequenceid=273, filesize=12.0 K 2024-11-23T16:13:55,035 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-23T16:13:55,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302. 2024-11-23T16:13:55,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302. as already flushing 2024-11-23T16:13:55,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302. 2024-11-23T16:13:55,036 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 4550087eb34ce466d194f9ce96539302, NAME => 'TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:13:55,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 4550087eb34ce466d194f9ce96539302, NAME => 'TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:13:55,037 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~375.56 KB/384576, currentSize=46.96 KB/48090 for 4550087eb34ce466d194f9ce96539302 in 893ms, sequenceid=273, compaction requested=true 2024-11-23T16:13:55,037 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4550087eb34ce466d194f9ce96539302: 2024-11-23T16:13:55,037 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4550087eb34ce466d194f9ce96539302:A, priority=-2147483648, current under compaction store size is 1 2024-11-23T16:13:55,037 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-23T16:13:55,037 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T16:13:55,037 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-23T16:13:55,037 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4550087eb34ce466d194f9ce96539302:B, priority=-2147483648, current under compaction store size is 2 2024-11-23T16:13:55,038 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T16:13:55,038 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4550087eb34ce466d194f9ce96539302:C, priority=-2147483648, current under compaction store size is 3 2024-11-23T16:13:55,038 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T16:13:55,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45799 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4550087eb34ce466d194f9ce96539302, NAME => 'TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4550087eb34ce466d194f9ce96539302, NAME => 'TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:13:55,039 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 56418 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-23T16:13:55,039 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HStore(1541): 4550087eb34ce466d194f9ce96539302/A is initiating minor compaction (all files) 2024-11-23T16:13:55,039 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4550087eb34ce466d194f9ce96539302/A in TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302. 2024-11-23T16:13:55,039 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/A/6caa843f43804b5b838656e781bfd55a, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/A/4f136979e1eb43d1a21d5f90e040a57a, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/A/7f44ac7f49c542b1bf09ac3260d5a75c, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/A/7b655553f25d453683d972d6ddb07d66] into tmpdir=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp, totalSize=55.1 K 2024-11-23T16:13:55,041 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49198 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-23T16:13:55,041 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HStore(1541): 4550087eb34ce466d194f9ce96539302/B is initiating minor compaction (all files) 2024-11-23T16:13:55,041 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4550087eb34ce466d194f9ce96539302/B in TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302. 2024-11-23T16:13:55,041 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/B/92d606bbe87a4d2ca12c45d4f9e9bd80, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/B/4bb8355660b944d6ad482d53cb9d269f, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/B/2f5bc73e426a4ab4835bb733a76492e9, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/B/237cc2230d8347239f11d37f868210a8] into tmpdir=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp, totalSize=48.0 K 2024-11-23T16:13:55,041 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6caa843f43804b5b838656e781bfd55a, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=203, earliestPutTs=1732378430847 2024-11-23T16:13:55,041 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.Compactor(225): Compacting 92d606bbe87a4d2ca12c45d4f9e9bd80, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=203, earliestPutTs=1732378430847 2024-11-23T16:13:55,042 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4f136979e1eb43d1a21d5f90e040a57a, keycount=250, bloomtype=ROW, size=16.5 K, encoding=NONE, compression=NONE, seqNum=225, earliestPutTs=1732378432087 2024-11-23T16:13:55,042 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.Compactor(225): Compacting 4bb8355660b944d6ad482d53cb9d269f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=225, earliestPutTs=1732378432087 2024-11-23T16:13:55,042 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7f44ac7f49c542b1bf09ac3260d5a75c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1732378432758 2024-11-23T16:13:55,043 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.Compactor(225): Compacting 2f5bc73e426a4ab4835bb733a76492e9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1732378432758 2024-11-23T16:13:55,043 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7b655553f25d453683d972d6ddb07d66, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=273, earliestPutTs=1732378433484 2024-11-23T16:13:55,044 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.Compactor(225): Compacting 237cc2230d8347239f11d37f868210a8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=273, earliestPutTs=1732378433494 2024-11-23T16:13:55,061 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4550087eb34ce466d194f9ce96539302#B#compaction#51 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T16:13:55,062 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp/B/0bec775274dd4ebea8ea426b7454137e is 50, key is test_row_0/B:col10/1732378434141/Put/seqid=0 2024-11-23T16:13:55,073 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4550087eb34ce466d194f9ce96539302#A#compaction#52 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T16:13:55,074 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp/A/98da1ac6baa647e2b68406fa88e32244 is 50, key is test_row_0/A:col10/1732378434141/Put/seqid=0 2024-11-23T16:13:55,086 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073741888_1064 (size=12881) 2024-11-23T16:13:55,097 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073741889_1065 (size=12881) 2024-11-23T16:13:55,103 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp/B/0bec775274dd4ebea8ea426b7454137e as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/B/0bec775274dd4ebea8ea426b7454137e 2024-11-23T16:13:55,113 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 4550087eb34ce466d194f9ce96539302/B of 4550087eb34ce466d194f9ce96539302 into 0bec775274dd4ebea8ea426b7454137e(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T16:13:55,113 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4550087eb34ce466d194f9ce96539302: 2024-11-23T16:13:55,113 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., storeName=4550087eb34ce466d194f9ce96539302/B, priority=12, startTime=1732378435037; duration=0sec 2024-11-23T16:13:55,113 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T16:13:55,113 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4550087eb34ce466d194f9ce96539302:B 2024-11-23T16:13:55,113 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-23T16:13:55,119 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49198 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-23T16:13:55,119 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HStore(1541): 4550087eb34ce466d194f9ce96539302/C is initiating minor compaction (all files) 2024-11-23T16:13:55,119 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4550087eb34ce466d194f9ce96539302/C in TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302. 2024-11-23T16:13:55,120 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/C/37dc0b6979544dcb873b925da413dc17, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/C/22c4912c178e436d89d44351fd1301d5, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/C/9d94f4df87b142cd9d4b3fc9571fe4d0, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/C/b9fd72e4d8d742839a4e929b4e5943d3] into tmpdir=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp, totalSize=48.0 K 2024-11-23T16:13:55,121 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.Compactor(225): Compacting 37dc0b6979544dcb873b925da413dc17, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=203, earliestPutTs=1732378430847 2024-11-23T16:13:55,121 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.Compactor(225): Compacting 22c4912c178e436d89d44351fd1301d5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=225, earliestPutTs=1732378432087 2024-11-23T16:13:55,122 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.Compactor(225): Compacting 9d94f4df87b142cd9d4b3fc9571fe4d0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1732378432758 2024-11-23T16:13:55,122 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.Compactor(225): Compacting b9fd72e4d8d742839a4e929b4e5943d3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=273, earliestPutTs=1732378433494 2024-11-23T16:13:55,139 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4550087eb34ce466d194f9ce96539302#C#compaction#53 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T16:13:55,140 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp/C/1843f289f90e4db693c0c6bc0387dcf5 is 50, key is test_row_0/C:col10/1732378434141/Put/seqid=0 2024-11-23T16:13:55,147 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073741890_1066 (size=12881) 2024-11-23T16:13:55,161 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp/C/1843f289f90e4db693c0c6bc0387dcf5 as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/C/1843f289f90e4db693c0c6bc0387dcf5 2024-11-23T16:13:55,171 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 4550087eb34ce466d194f9ce96539302/C of 4550087eb34ce466d194f9ce96539302 into 1843f289f90e4db693c0c6bc0387dcf5(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T16:13:55,171 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4550087eb34ce466d194f9ce96539302: 2024-11-23T16:13:55,171 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., storeName=4550087eb34ce466d194f9ce96539302/C, priority=12, startTime=1732378435038; duration=0sec 2024-11-23T16:13:55,171 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T16:13:55,171 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4550087eb34ce466d194f9ce96539302:C 2024-11-23T16:13:55,192 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-23T16:13:55,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302. 2024-11-23T16:13:55,193 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2902): Flushing 4550087eb34ce466d194f9ce96539302 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-23T16:13:55,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4550087eb34ce466d194f9ce96539302, store=A 2024-11-23T16:13:55,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-23T16:13:55,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4550087eb34ce466d194f9ce96539302, store=B 2024-11-23T16:13:55,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-23T16:13:55,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4550087eb34ce466d194f9ce96539302, store=C 2024-11-23T16:13:55,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-23T16:13:55,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-23T16:13:55,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp/A/22598940ec8d4369a016eef57a7b26bd is 50, key is test_row_0/A:col10/1732378434183/Put/seqid=0 2024-11-23T16:13:55,219 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073741891_1067 (size=9857) 2024-11-23T16:13:55,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(8855): Flush requested on 4550087eb34ce466d194f9ce96539302 2024-11-23T16:13:55,342 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302. as already flushing 2024-11-23T16:13:55,393 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1a7c5470b472:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732378435364, max timestamp=1732378435389Num uniques 859; 2024-11-23T16:13:55,393 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1a7c5470b472:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-23T16:13:55,393 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1a7c5470b472:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732378435364, max timestamp=1732378435389Num uniques 859; 2024-11-23T16:13:55,393 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1a7c5470b472:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732378435364, max timestamp=1732378435389Num uniques 859; 2024-11-23T16:13:55,393 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1a7c5470b472:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-23T16:13:55,393 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1a7c5470b472:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-23T16:13:55,428 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:13:55,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40416 deadline: 1732378495425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:13:55,429 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:55,429 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:55,429 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:13:55,429 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:13:55,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40410 deadline: 1732378495426, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:13:55,429 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:55,430 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:55,430 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:13:55,504 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp/A/98da1ac6baa647e2b68406fa88e32244 as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/A/98da1ac6baa647e2b68406fa88e32244 2024-11-23T16:13:55,511 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 4550087eb34ce466d194f9ce96539302/A of 4550087eb34ce466d194f9ce96539302 into 98da1ac6baa647e2b68406fa88e32244(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T16:13:55,511 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4550087eb34ce466d194f9ce96539302: 2024-11-23T16:13:55,511 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., storeName=4550087eb34ce466d194f9ce96539302/A, priority=12, startTime=1732378435037; duration=0sec 2024-11-23T16:13:55,511 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T16:13:55,511 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4550087eb34ce466d194f9ce96539302:A 2024-11-23T16:13:55,541 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:13:55,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40416 deadline: 1732378495538, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:13:55,542 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:55,542 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:13:55,542 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:55,542 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:13:55,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40410 deadline: 1732378495538, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:13:55,543 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:55,543 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:55,543 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:13:55,620 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=285 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp/A/22598940ec8d4369a016eef57a7b26bd 2024-11-23T16:13:55,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp/B/498aadee42ea493db429f2f27dad6dca is 50, key is test_row_0/B:col10/1732378434183/Put/seqid=0 2024-11-23T16:13:55,637 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073741892_1068 (size=9857) 2024-11-23T16:13:55,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-23T16:13:55,751 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:13:55,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40416 deadline: 1732378495748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:13:55,752 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:55,752 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:13:55,752 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:55,752 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:13:55,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40410 deadline: 1732378495749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:13:55,752 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:55,752 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:55,752 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:13:56,038 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=285 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp/B/498aadee42ea493db429f2f27dad6dca 2024-11-23T16:13:56,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp/C/79ac225079bc4bd6bb395773461f3f42 is 50, key is test_row_0/C:col10/1732378434183/Put/seqid=0 2024-11-23T16:13:56,053 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073741893_1069 (size=9857) 2024-11-23T16:13:56,058 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:13:56,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40416 deadline: 1732378496058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:13:56,059 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:56,059 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:56,059 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:13:56,059 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:13:56,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40410 deadline: 1732378496058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:13:56,060 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:56,060 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:56,060 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:13:56,454 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=285 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp/C/79ac225079bc4bd6bb395773461f3f42 2024-11-23T16:13:56,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp/A/22598940ec8d4369a016eef57a7b26bd as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/A/22598940ec8d4369a016eef57a7b26bd 2024-11-23T16:13:56,467 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/A/22598940ec8d4369a016eef57a7b26bd, entries=100, sequenceid=285, filesize=9.6 K 2024-11-23T16:13:56,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp/B/498aadee42ea493db429f2f27dad6dca as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/B/498aadee42ea493db429f2f27dad6dca 2024-11-23T16:13:56,473 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/B/498aadee42ea493db429f2f27dad6dca, entries=100, sequenceid=285, filesize=9.6 K 2024-11-23T16:13:56,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp/C/79ac225079bc4bd6bb395773461f3f42 as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/C/79ac225079bc4bd6bb395773461f3f42 2024-11-23T16:13:56,479 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/C/79ac225079bc4bd6bb395773461f3f42, entries=100, sequenceid=285, filesize=9.6 K 2024-11-23T16:13:56,480 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 4550087eb34ce466d194f9ce96539302 in 1287ms, sequenceid=285, compaction requested=false 2024-11-23T16:13:56,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2603): Flush status journal for 4550087eb34ce466d194f9ce96539302: 2024-11-23T16:13:56,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302. 2024-11-23T16:13:56,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=16 2024-11-23T16:13:56,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45799 {}] master.HMaster(4169): Remote procedure done, pid=16 2024-11-23T16:13:56,485 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=15 2024-11-23T16:13:56,485 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9080 sec 2024-11-23T16:13:56,488 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees in 1.9170 sec 2024-11-23T16:13:56,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(8855): Flush requested on 4550087eb34ce466d194f9ce96539302 2024-11-23T16:13:56,574 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4550087eb34ce466d194f9ce96539302 3/3 column families, dataSize=140.89 KB heapSize=395.53 KB 2024-11-23T16:13:56,574 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4550087eb34ce466d194f9ce96539302, store=A 2024-11-23T16:13:56,574 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-23T16:13:56,574 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4550087eb34ce466d194f9ce96539302, store=B 2024-11-23T16:13:56,574 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-23T16:13:56,574 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4550087eb34ce466d194f9ce96539302, store=C 2024-11-23T16:13:56,574 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-23T16:13:56,581 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp/A/361f8b365cd04c05af07ccabf074cbb8 is 50, key is test_row_0/A:col10/1732378435422/Put/seqid=0 2024-11-23T16:13:56,586 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073741894_1070 (size=14741) 2024-11-23T16:13:56,591 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:13:56,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40410 deadline: 1732378496582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:13:56,592 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:56,592 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:56,592 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:13:56,592 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:13:56,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40416 deadline: 1732378496588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:13:56,592 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:56,593 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:56,593 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:13:56,699 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:13:56,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40410 deadline: 1732378496698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:13:56,700 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:13:56,700 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:56,700 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:56,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40416 deadline: 1732378496699, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:13:56,700 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:13:56,700 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:56,700 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:56,700 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:13:56,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-23T16:13:56,718 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-23T16:13:56,719 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45799 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-23T16:13:56,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45799 {}] procedure2.ProcedureExecutor(1139): Stored pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees 2024-11-23T16:13:56,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-23T16:13:56,721 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-23T16:13:56,722 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-23T16:13:56,722 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-23T16:13:56,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-23T16:13:56,876 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-23T16:13:56,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302. 2024-11-23T16:13:56,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302. as already flushing 2024-11-23T16:13:56,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302. 2024-11-23T16:13:56,876 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 4550087eb34ce466d194f9ce96539302, NAME => 'TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:13:56,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 4550087eb34ce466d194f9ce96539302, NAME => 'TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:13:56,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45799 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4550087eb34ce466d194f9ce96539302, NAME => 'TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4550087eb34ce466d194f9ce96539302, NAME => 'TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:13:56,908 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:13:56,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40410 deadline: 1732378496908, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:13:56,909 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:56,909 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:13:56,909 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:56,909 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:13:56,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40416 deadline: 1732378496908, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:13:56,910 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:56,910 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:56,910 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:13:56,987 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=322 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp/A/361f8b365cd04c05af07ccabf074cbb8 2024-11-23T16:13:56,999 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp/B/dd0035fc11d141708484369b5f066f29 is 50, key is test_row_0/B:col10/1732378435422/Put/seqid=0 2024-11-23T16:13:57,011 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073741895_1071 (size=12301) 2024-11-23T16:13:57,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-23T16:13:57,050 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-23T16:13:57,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302. 2024-11-23T16:13:57,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302. as already flushing 2024-11-23T16:13:57,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302. 2024-11-23T16:13:57,050 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 4550087eb34ce466d194f9ce96539302, NAME => 'TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:13:57,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 4550087eb34ce466d194f9ce96539302, NAME => 'TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:13:57,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45799 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4550087eb34ce466d194f9ce96539302, NAME => 'TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4550087eb34ce466d194f9ce96539302, NAME => 'TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:13:57,204 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-23T16:13:57,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302. 2024-11-23T16:13:57,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302. as already flushing 2024-11-23T16:13:57,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302. 2024-11-23T16:13:57,205 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 4550087eb34ce466d194f9ce96539302, NAME => 'TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:13:57,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 4550087eb34ce466d194f9ce96539302, NAME => 'TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:13:57,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45799 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4550087eb34ce466d194f9ce96539302, NAME => 'TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4550087eb34ce466d194f9ce96539302, NAME => 'TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:13:57,223 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:13:57,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40410 deadline: 1732378497218, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:13:57,224 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:13:57,224 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:57,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40416 deadline: 1732378497218, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:13:57,224 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:57,224 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:13:57,224 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:57,225 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:57,225 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:13:57,357 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-23T16:13:57,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302. 2024-11-23T16:13:57,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302. as already flushing 2024-11-23T16:13:57,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302. 2024-11-23T16:13:57,358 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 4550087eb34ce466d194f9ce96539302, NAME => 'TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:13:57,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-23T16:13:57,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 4550087eb34ce466d194f9ce96539302, NAME => 'TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:13:57,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45799 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4550087eb34ce466d194f9ce96539302, NAME => 'TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4550087eb34ce466d194f9ce96539302, NAME => 'TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:13:57,412 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=322 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp/B/dd0035fc11d141708484369b5f066f29 2024-11-23T16:13:57,424 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp/C/ebd861dd1bc440c4a7f62684ebd02003 is 50, key is test_row_0/C:col10/1732378435422/Put/seqid=0 2024-11-23T16:13:57,429 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073741896_1072 (size=12301) 2024-11-23T16:13:57,511 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-23T16:13:57,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302. 2024-11-23T16:13:57,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302. as already flushing 2024-11-23T16:13:57,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302. 2024-11-23T16:13:57,512 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 4550087eb34ce466d194f9ce96539302, NAME => 'TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:13:57,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 4550087eb34ce466d194f9ce96539302, NAME => 'TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:13:57,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45799 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4550087eb34ce466d194f9ce96539302, NAME => 'TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4550087eb34ce466d194f9ce96539302, NAME => 'TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:13:57,540 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:13:57,540 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:13:57,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40390 deadline: 1732378497538, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:13:57,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40380 deadline: 1732378497538, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:13:57,541 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 1a7c5470b472,44715,1732378420375 for 'test_row_1' in 4550087eb34ce466d194f9ce96539302 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8238 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-23T16:13:57,541 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 1a7c5470b472,44715,1732378420375 for 'test_row_0' in 4550087eb34ce466d194f9ce96539302 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8240 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-23T16:13:57,542 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:57,542 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:57,542 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:57,542 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:57,542 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:13:57,542 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:13:57,553 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:13:57,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40350 deadline: 1732378497548, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:13:57,554 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 1a7c5470b472,44715,1732378420375 for 'test_row_2' in 4550087eb34ce466d194f9ce96539302 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8251 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-23T16:13:57,554 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:57,554 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:57,554 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:13:57,665 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-23T16:13:57,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302. 2024-11-23T16:13:57,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302. as already flushing 2024-11-23T16:13:57,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302. 2024-11-23T16:13:57,666 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 4550087eb34ce466d194f9ce96539302, NAME => 'TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:13:57,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 4550087eb34ce466d194f9ce96539302, NAME => 'TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:13:57,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45799 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4550087eb34ce466d194f9ce96539302, NAME => 'TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4550087eb34ce466d194f9ce96539302, NAME => 'TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:13:57,742 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:13:57,742 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:13:57,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40410 deadline: 1732378497738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:13:57,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40416 deadline: 1732378497738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:13:57,743 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:57,743 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:57,743 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:57,743 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:13:57,743 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:57,743 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:13:57,820 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-23T16:13:57,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302. 2024-11-23T16:13:57,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302. as already flushing 2024-11-23T16:13:57,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302. 2024-11-23T16:13:57,820 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 4550087eb34ce466d194f9ce96539302, NAME => 'TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:13:57,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 4550087eb34ce466d194f9ce96539302, NAME => 'TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:13:57,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45799 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4550087eb34ce466d194f9ce96539302, NAME => 'TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4550087eb34ce466d194f9ce96539302, NAME => 'TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:13:57,830 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=322 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp/C/ebd861dd1bc440c4a7f62684ebd02003 2024-11-23T16:13:57,836 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp/A/361f8b365cd04c05af07ccabf074cbb8 as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/A/361f8b365cd04c05af07ccabf074cbb8 2024-11-23T16:13:57,842 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/A/361f8b365cd04c05af07ccabf074cbb8, entries=200, sequenceid=322, filesize=14.4 K 2024-11-23T16:13:57,844 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp/B/dd0035fc11d141708484369b5f066f29 as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/B/dd0035fc11d141708484369b5f066f29 2024-11-23T16:13:57,850 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/B/dd0035fc11d141708484369b5f066f29, entries=150, sequenceid=322, filesize=12.0 K 2024-11-23T16:13:57,851 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp/C/ebd861dd1bc440c4a7f62684ebd02003 as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/C/ebd861dd1bc440c4a7f62684ebd02003 2024-11-23T16:13:57,859 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/C/ebd861dd1bc440c4a7f62684ebd02003, entries=150, sequenceid=322, filesize=12.0 K 2024-11-23T16:13:57,861 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~395.48 KB/404976, currentSize=40.25 KB/41220 for 4550087eb34ce466d194f9ce96539302 in 1286ms, sequenceid=322, compaction requested=true 2024-11-23T16:13:57,861 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4550087eb34ce466d194f9ce96539302: 2024-11-23T16:13:57,861 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4550087eb34ce466d194f9ce96539302:A, priority=-2147483648, current under compaction store size is 1 2024-11-23T16:13:57,861 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T16:13:57,861 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T16:13:57,861 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4550087eb34ce466d194f9ce96539302:B, priority=-2147483648, current under compaction store size is 2 2024-11-23T16:13:57,861 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T16:13:57,861 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T16:13:57,861 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4550087eb34ce466d194f9ce96539302:C, priority=-2147483648, current under compaction store size is 3 2024-11-23T16:13:57,861 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T16:13:57,863 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37479 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T16:13:57,863 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HStore(1541): 4550087eb34ce466d194f9ce96539302/A is initiating minor compaction (all files) 2024-11-23T16:13:57,863 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4550087eb34ce466d194f9ce96539302/A in TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302. 2024-11-23T16:13:57,863 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/A/98da1ac6baa647e2b68406fa88e32244, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/A/22598940ec8d4369a016eef57a7b26bd, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/A/361f8b365cd04c05af07ccabf074cbb8] into tmpdir=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp, totalSize=36.6 K 2024-11-23T16:13:57,864 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35039 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T16:13:57,864 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HStore(1541): 4550087eb34ce466d194f9ce96539302/B is initiating minor compaction (all files) 2024-11-23T16:13:57,864 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.Compactor(225): Compacting 98da1ac6baa647e2b68406fa88e32244, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=273, earliestPutTs=1732378433494 2024-11-23T16:13:57,864 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4550087eb34ce466d194f9ce96539302/B in TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302. 2024-11-23T16:13:57,864 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/B/0bec775274dd4ebea8ea426b7454137e, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/B/498aadee42ea493db429f2f27dad6dca, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/B/dd0035fc11d141708484369b5f066f29] into tmpdir=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp, totalSize=34.2 K 2024-11-23T16:13:57,864 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.Compactor(225): Compacting 22598940ec8d4369a016eef57a7b26bd, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=285, earliestPutTs=1732378434183 2024-11-23T16:13:57,864 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.Compactor(225): Compacting 0bec775274dd4ebea8ea426b7454137e, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=273, earliestPutTs=1732378433494 2024-11-23T16:13:57,865 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.Compactor(225): Compacting 498aadee42ea493db429f2f27dad6dca, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=285, earliestPutTs=1732378434183 2024-11-23T16:13:57,865 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.Compactor(225): Compacting 361f8b365cd04c05af07ccabf074cbb8, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=322, earliestPutTs=1732378435422 2024-11-23T16:13:57,865 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.Compactor(225): Compacting dd0035fc11d141708484369b5f066f29, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=322, earliestPutTs=1732378435422 2024-11-23T16:13:57,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-23T16:13:57,876 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4550087eb34ce466d194f9ce96539302#B#compaction#60 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-23T16:13:57,877 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp/B/08b3dc087e664d13b2c7cae2252b1ddf is 50, key is test_row_0/B:col10/1732378435422/Put/seqid=0 2024-11-23T16:13:57,878 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4550087eb34ce466d194f9ce96539302#A#compaction#61 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T16:13:57,879 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp/A/0abf882ec4b54304a78ed6a8c8005e4a is 50, key is test_row_0/A:col10/1732378435422/Put/seqid=0 2024-11-23T16:13:57,892 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073741897_1073 (size=12983) 2024-11-23T16:13:57,894 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073741898_1074 (size=12983) 2024-11-23T16:13:57,903 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp/A/0abf882ec4b54304a78ed6a8c8005e4a as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/A/0abf882ec4b54304a78ed6a8c8005e4a 2024-11-23T16:13:57,903 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp/B/08b3dc087e664d13b2c7cae2252b1ddf as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/B/08b3dc087e664d13b2c7cae2252b1ddf 2024-11-23T16:13:57,913 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4550087eb34ce466d194f9ce96539302/A of 4550087eb34ce466d194f9ce96539302 into 0abf882ec4b54304a78ed6a8c8005e4a(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T16:13:57,913 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4550087eb34ce466d194f9ce96539302: 2024-11-23T16:13:57,914 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., storeName=4550087eb34ce466d194f9ce96539302/A, priority=13, startTime=1732378437861; duration=0sec 2024-11-23T16:13:57,915 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T16:13:57,915 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4550087eb34ce466d194f9ce96539302:A 2024-11-23T16:13:57,915 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T16:13:57,917 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35039 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T16:13:57,917 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HStore(1541): 4550087eb34ce466d194f9ce96539302/C is initiating minor compaction (all files) 2024-11-23T16:13:57,917 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4550087eb34ce466d194f9ce96539302/C in TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302. 2024-11-23T16:13:57,917 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/C/1843f289f90e4db693c0c6bc0387dcf5, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/C/79ac225079bc4bd6bb395773461f3f42, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/C/ebd861dd1bc440c4a7f62684ebd02003] into tmpdir=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp, totalSize=34.2 K 2024-11-23T16:13:57,917 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4550087eb34ce466d194f9ce96539302/B of 4550087eb34ce466d194f9ce96539302 into 08b3dc087e664d13b2c7cae2252b1ddf(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T16:13:57,917 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4550087eb34ce466d194f9ce96539302: 2024-11-23T16:13:57,918 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., storeName=4550087eb34ce466d194f9ce96539302/B, priority=13, startTime=1732378437861; duration=0sec 2024-11-23T16:13:57,918 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T16:13:57,918 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4550087eb34ce466d194f9ce96539302:B 2024-11-23T16:13:57,918 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1843f289f90e4db693c0c6bc0387dcf5, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=273, earliestPutTs=1732378433494 2024-11-23T16:13:57,919 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.Compactor(225): Compacting 79ac225079bc4bd6bb395773461f3f42, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=285, earliestPutTs=1732378434183 2024-11-23T16:13:57,919 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.Compactor(225): Compacting ebd861dd1bc440c4a7f62684ebd02003, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=322, earliestPutTs=1732378435422 2024-11-23T16:13:57,932 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4550087eb34ce466d194f9ce96539302#C#compaction#62 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T16:13:57,933 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp/C/85fcbe1353a1428dbf47aaf4aad9b51b is 50, key is test_row_0/C:col10/1732378435422/Put/seqid=0 2024-11-23T16:13:57,943 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073741899_1075 (size=12983) 2024-11-23T16:13:57,974 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-23T16:13:57,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302. 2024-11-23T16:13:57,974 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2902): Flushing 4550087eb34ce466d194f9ce96539302 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-11-23T16:13:57,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4550087eb34ce466d194f9ce96539302, store=A 2024-11-23T16:13:57,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-23T16:13:57,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4550087eb34ce466d194f9ce96539302, store=B 2024-11-23T16:13:57,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-23T16:13:57,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4550087eb34ce466d194f9ce96539302, store=C 2024-11-23T16:13:57,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-23T16:13:57,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp/A/c8d72f03685c415a8dbc665ce04aa59e is 50, key is test_row_0/A:col10/1732378436586/Put/seqid=0 2024-11-23T16:13:57,985 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073741900_1076 (size=12301) 2024-11-23T16:13:57,986 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=333 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp/A/c8d72f03685c415a8dbc665ce04aa59e 2024-11-23T16:13:57,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp/B/a831559cdf8d48b8888617f1ea7fccbf is 50, key is test_row_0/B:col10/1732378436586/Put/seqid=0 2024-11-23T16:13:58,010 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073741901_1077 (size=12301) 2024-11-23T16:13:58,011 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=333 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp/B/a831559cdf8d48b8888617f1ea7fccbf 2024-11-23T16:13:58,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp/C/3a9cdbf40e28420f929e60a9ed7f3463 is 50, key is test_row_0/C:col10/1732378436586/Put/seqid=0 2024-11-23T16:13:58,029 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073741902_1078 (size=12301) 2024-11-23T16:13:58,352 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp/C/85fcbe1353a1428dbf47aaf4aad9b51b as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/C/85fcbe1353a1428dbf47aaf4aad9b51b 2024-11-23T16:13:58,365 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4550087eb34ce466d194f9ce96539302/C of 4550087eb34ce466d194f9ce96539302 into 85fcbe1353a1428dbf47aaf4aad9b51b(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T16:13:58,365 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4550087eb34ce466d194f9ce96539302: 2024-11-23T16:13:58,365 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., storeName=4550087eb34ce466d194f9ce96539302/C, priority=13, startTime=1732378437861; duration=0sec 2024-11-23T16:13:58,366 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T16:13:58,366 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4550087eb34ce466d194f9ce96539302:C 2024-11-23T16:13:58,431 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=333 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp/C/3a9cdbf40e28420f929e60a9ed7f3463 2024-11-23T16:13:58,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp/A/c8d72f03685c415a8dbc665ce04aa59e as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/A/c8d72f03685c415a8dbc665ce04aa59e 2024-11-23T16:13:58,454 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/A/c8d72f03685c415a8dbc665ce04aa59e, entries=150, sequenceid=333, filesize=12.0 K 2024-11-23T16:13:58,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp/B/a831559cdf8d48b8888617f1ea7fccbf as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/B/a831559cdf8d48b8888617f1ea7fccbf 2024-11-23T16:13:58,462 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/B/a831559cdf8d48b8888617f1ea7fccbf, entries=150, sequenceid=333, filesize=12.0 K 2024-11-23T16:13:58,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp/C/3a9cdbf40e28420f929e60a9ed7f3463 as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/C/3a9cdbf40e28420f929e60a9ed7f3463 2024-11-23T16:13:58,478 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/C/3a9cdbf40e28420f929e60a9ed7f3463, entries=150, sequenceid=333, filesize=12.0 K 2024-11-23T16:13:58,480 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=0 B/0 for 4550087eb34ce466d194f9ce96539302 in 506ms, sequenceid=333, compaction requested=false 2024-11-23T16:13:58,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2603): Flush status journal for 4550087eb34ce466d194f9ce96539302: 2024-11-23T16:13:58,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302. 2024-11-23T16:13:58,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=18 2024-11-23T16:13:58,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45799 {}] master.HMaster(4169): Remote procedure done, pid=18 2024-11-23T16:13:58,488 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=17 2024-11-23T16:13:58,488 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7610 sec 2024-11-23T16:13:58,491 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees in 1.7700 sec 2024-11-23T16:13:58,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(8855): Flush requested on 4550087eb34ce466d194f9ce96539302 2024-11-23T16:13:58,765 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4550087eb34ce466d194f9ce96539302 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-23T16:13:58,765 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4550087eb34ce466d194f9ce96539302, store=A 2024-11-23T16:13:58,765 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-23T16:13:58,765 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4550087eb34ce466d194f9ce96539302, store=B 2024-11-23T16:13:58,766 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-23T16:13:58,766 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4550087eb34ce466d194f9ce96539302, store=C 2024-11-23T16:13:58,766 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-23T16:13:58,789 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp/A/38c4d67e5e68480b9a4b0a689a6ad87a is 50, key is test_row_0/A:col10/1732378438763/Put/seqid=0 2024-11-23T16:13:58,820 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073741903_1079 (size=14737) 2024-11-23T16:13:58,821 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=345 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp/A/38c4d67e5e68480b9a4b0a689a6ad87a 2024-11-23T16:13:58,837 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1a7c5470b472:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732378438817, max timestamp=1732378438832Num uniques 859; 2024-11-23T16:13:58,837 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1a7c5470b472:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732378438817, max timestamp=1732378438832Num uniques 859; 2024-11-23T16:13:58,837 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1a7c5470b472:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-23T16:13:58,837 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1a7c5470b472:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-23T16:13:58,837 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1a7c5470b472:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732378438817, max timestamp=1732378438832Num uniques 859; 2024-11-23T16:13:58,837 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1a7c5470b472:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-23T16:13:58,842 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp/B/d5b0a45ddba84a27bb69b39717d3cef5 is 50, key is test_row_0/B:col10/1732378438763/Put/seqid=0 2024-11-23T16:13:58,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-23T16:13:58,883 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-23T16:13:58,884 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45799 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-23T16:13:58,886 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073741904_1080 (size=9857) 2024-11-23T16:13:58,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45799 {}] procedure2.ProcedureExecutor(1139): Stored pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees 2024-11-23T16:13:58,888 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-23T16:13:58,890 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-23T16:13:58,890 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-23T16:13:58,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-23T16:13:58,906 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:13:58,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40410 deadline: 1732378498897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:13:58,907 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:58,907 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:58,907 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:13:58,909 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:13:58,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 181 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40416 deadline: 1732378498904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:13:58,910 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:58,910 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:58,910 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:13:58,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-23T16:13:59,021 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:13:59,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40410 deadline: 1732378499019, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:13:59,022 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:13:59,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 182 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40416 deadline: 1732378499019, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:13:59,022 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:59,022 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:59,022 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:13:59,024 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:59,024 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:59,024 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:13:59,044 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-23T16:13:59,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302. 2024-11-23T16:13:59,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302. as already flushing 2024-11-23T16:13:59,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302. 2024-11-23T16:13:59,045 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 4550087eb34ce466d194f9ce96539302, NAME => 'TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:13:59,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 4550087eb34ce466d194f9ce96539302, NAME => 'TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:13:59,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45799 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4550087eb34ce466d194f9ce96539302, NAME => 'TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4550087eb34ce466d194f9ce96539302, NAME => 'TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:13:59,199 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-23T16:13:59,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302. 2024-11-23T16:13:59,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302. as already flushing 2024-11-23T16:13:59,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302. 2024-11-23T16:13:59,200 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 4550087eb34ce466d194f9ce96539302, NAME => 'TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:13:59,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 4550087eb34ce466d194f9ce96539302, NAME => 'TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:13:59,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45799 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4550087eb34ce466d194f9ce96539302, NAME => 'TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4550087eb34ce466d194f9ce96539302, NAME => 'TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:13:59,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-23T16:13:59,228 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:13:59,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40410 deadline: 1732378499228, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:13:59,229 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:13:59,229 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:59,229 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:59,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 183 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40416 deadline: 1732378499228, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:13:59,229 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:13:59,230 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:59,230 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:59,230 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:13:59,287 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=345 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp/B/d5b0a45ddba84a27bb69b39717d3cef5 2024-11-23T16:13:59,300 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp/C/57bab8993a1d45d6be4ba7f27b12e58e is 50, key is test_row_0/C:col10/1732378438763/Put/seqid=0 2024-11-23T16:13:59,311 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073741905_1081 (size=9857) 2024-11-23T16:13:59,312 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=345 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp/C/57bab8993a1d45d6be4ba7f27b12e58e 2024-11-23T16:13:59,319 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp/A/38c4d67e5e68480b9a4b0a689a6ad87a as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/A/38c4d67e5e68480b9a4b0a689a6ad87a 2024-11-23T16:13:59,333 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/A/38c4d67e5e68480b9a4b0a689a6ad87a, entries=200, sequenceid=345, filesize=14.4 K 2024-11-23T16:13:59,335 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp/B/d5b0a45ddba84a27bb69b39717d3cef5 as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/B/d5b0a45ddba84a27bb69b39717d3cef5 2024-11-23T16:13:59,342 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/B/d5b0a45ddba84a27bb69b39717d3cef5, entries=100, sequenceid=345, filesize=9.6 K 2024-11-23T16:13:59,343 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp/C/57bab8993a1d45d6be4ba7f27b12e58e as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/C/57bab8993a1d45d6be4ba7f27b12e58e 2024-11-23T16:13:59,355 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/C/57bab8993a1d45d6be4ba7f27b12e58e, entries=100, sequenceid=345, filesize=9.6 K 2024-11-23T16:13:59,356 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 4550087eb34ce466d194f9ce96539302 in 591ms, sequenceid=345, compaction requested=true 2024-11-23T16:13:59,357 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4550087eb34ce466d194f9ce96539302: 2024-11-23T16:13:59,357 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4550087eb34ce466d194f9ce96539302:A, priority=-2147483648, current under compaction store size is 1 2024-11-23T16:13:59,357 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T16:13:59,357 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T16:13:59,357 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4550087eb34ce466d194f9ce96539302:B, priority=-2147483648, current under compaction store size is 2 2024-11-23T16:13:59,357 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T16:13:59,357 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T16:13:59,357 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4550087eb34ce466d194f9ce96539302:C, priority=-2147483648, current under compaction store size is 3 2024-11-23T16:13:59,357 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T16:13:59,358 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-23T16:13:59,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302. 2024-11-23T16:13:59,358 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2902): Flushing 4550087eb34ce466d194f9ce96539302 3/3 column families, dataSize=127.47 KB heapSize=365.06 KB 2024-11-23T16:13:59,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4550087eb34ce466d194f9ce96539302, store=A 2024-11-23T16:13:59,359 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40021 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T16:13:59,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-23T16:13:59,359 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HStore(1541): 4550087eb34ce466d194f9ce96539302/A is initiating minor compaction (all files) 2024-11-23T16:13:59,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4550087eb34ce466d194f9ce96539302, store=B 2024-11-23T16:13:59,359 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4550087eb34ce466d194f9ce96539302/A in TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302. 2024-11-23T16:13:59,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-23T16:13:59,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4550087eb34ce466d194f9ce96539302, store=C 2024-11-23T16:13:59,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-23T16:13:59,359 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/A/0abf882ec4b54304a78ed6a8c8005e4a, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/A/c8d72f03685c415a8dbc665ce04aa59e, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/A/38c4d67e5e68480b9a4b0a689a6ad87a] into tmpdir=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp, totalSize=39.1 K 2024-11-23T16:13:59,359 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35141 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T16:13:59,359 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HStore(1541): 4550087eb34ce466d194f9ce96539302/B is initiating minor compaction (all files) 2024-11-23T16:13:59,359 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4550087eb34ce466d194f9ce96539302/B in TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302. 2024-11-23T16:13:59,360 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0abf882ec4b54304a78ed6a8c8005e4a, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=322, earliestPutTs=1732378435422 2024-11-23T16:13:59,360 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/B/08b3dc087e664d13b2c7cae2252b1ddf, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/B/a831559cdf8d48b8888617f1ea7fccbf, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/B/d5b0a45ddba84a27bb69b39717d3cef5] into tmpdir=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp, totalSize=34.3 K 2024-11-23T16:13:59,360 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.Compactor(225): Compacting c8d72f03685c415a8dbc665ce04aa59e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=333, earliestPutTs=1732378436579 2024-11-23T16:13:59,360 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.Compactor(225): Compacting 08b3dc087e664d13b2c7cae2252b1ddf, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=322, earliestPutTs=1732378435422 2024-11-23T16:13:59,361 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.Compactor(225): Compacting 38c4d67e5e68480b9a4b0a689a6ad87a, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=345, earliestPutTs=1732378438756 2024-11-23T16:13:59,361 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.Compactor(225): Compacting a831559cdf8d48b8888617f1ea7fccbf, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=333, earliestPutTs=1732378436579 2024-11-23T16:13:59,362 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.Compactor(225): Compacting d5b0a45ddba84a27bb69b39717d3cef5, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=345, earliestPutTs=1732378438763 2024-11-23T16:13:59,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp/A/d0f5a1fdd6264a5db5af52ec7c477538 is 50, key is test_row_0/A:col10/1732378438895/Put/seqid=0 2024-11-23T16:13:59,387 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4550087eb34ce466d194f9ce96539302#A#compaction#70 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T16:13:59,387 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp/A/01693751cc6c4dedbba2ca6e53da143a is 50, key is test_row_0/A:col10/1732378438763/Put/seqid=0 2024-11-23T16:13:59,391 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4550087eb34ce466d194f9ce96539302#B#compaction#71 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T16:13:59,392 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp/B/7e2d25392d2147289fc05b625c6ffd7e is 50, key is test_row_0/B:col10/1732378438763/Put/seqid=0 2024-11-23T16:13:59,422 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073741906_1082 (size=12301) 2024-11-23T16:13:59,441 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073741907_1083 (size=13085) 2024-11-23T16:13:59,458 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073741908_1084 (size=13085) 2024-11-23T16:13:59,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-23T16:13:59,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(8855): Flush requested on 4550087eb34ce466d194f9ce96539302 2024-11-23T16:13:59,542 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302. as already flushing 2024-11-23T16:13:59,571 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:13:59,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40410 deadline: 1732378499561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:13:59,572 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:59,572 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:59,572 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:13:59,574 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:13:59,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 188 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40416 deadline: 1732378499565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:13:59,575 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:59,575 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:59,575 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:13:59,681 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:13:59,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40410 deadline: 1732378499678, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:13:59,682 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:59,682 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:13:59,682 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:59,682 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:13:59,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 189 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40416 deadline: 1732378499678, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:13:59,683 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:59,683 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:59,683 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:13:59,823 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=381 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp/A/d0f5a1fdd6264a5db5af52ec7c477538 2024-11-23T16:13:59,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp/B/99ff1a763d9d4a1db56eb883ff76d25c is 50, key is test_row_0/B:col10/1732378438895/Put/seqid=0 2024-11-23T16:13:59,841 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073741909_1085 (size=12301) 2024-11-23T16:13:59,843 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=381 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp/B/99ff1a763d9d4a1db56eb883ff76d25c 2024-11-23T16:13:59,855 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp/A/01693751cc6c4dedbba2ca6e53da143a as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/A/01693751cc6c4dedbba2ca6e53da143a 2024-11-23T16:13:59,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp/C/39997ef645c4489c8dcd86a1b772f498 is 50, key is test_row_0/C:col10/1732378438895/Put/seqid=0 2024-11-23T16:13:59,864 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4550087eb34ce466d194f9ce96539302/A of 4550087eb34ce466d194f9ce96539302 into 01693751cc6c4dedbba2ca6e53da143a(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T16:13:59,864 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4550087eb34ce466d194f9ce96539302: 2024-11-23T16:13:59,865 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., storeName=4550087eb34ce466d194f9ce96539302/A, priority=13, startTime=1732378439357; duration=0sec 2024-11-23T16:13:59,865 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T16:13:59,865 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4550087eb34ce466d194f9ce96539302:A 2024-11-23T16:13:59,865 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T16:13:59,866 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35141 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T16:13:59,866 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HStore(1541): 4550087eb34ce466d194f9ce96539302/C is initiating minor compaction (all files) 2024-11-23T16:13:59,867 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4550087eb34ce466d194f9ce96539302/C in TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302. 2024-11-23T16:13:59,867 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/C/85fcbe1353a1428dbf47aaf4aad9b51b, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/C/3a9cdbf40e28420f929e60a9ed7f3463, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/C/57bab8993a1d45d6be4ba7f27b12e58e] into tmpdir=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp, totalSize=34.3 K 2024-11-23T16:13:59,868 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.Compactor(225): Compacting 85fcbe1353a1428dbf47aaf4aad9b51b, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=322, earliestPutTs=1732378435422 2024-11-23T16:13:59,869 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3a9cdbf40e28420f929e60a9ed7f3463, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=333, earliestPutTs=1732378436579 2024-11-23T16:13:59,870 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.Compactor(225): Compacting 57bab8993a1d45d6be4ba7f27b12e58e, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=345, earliestPutTs=1732378438763 2024-11-23T16:13:59,876 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp/B/7e2d25392d2147289fc05b625c6ffd7e as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/B/7e2d25392d2147289fc05b625c6ffd7e 2024-11-23T16:13:59,880 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073741910_1086 (size=12301) 2024-11-23T16:13:59,882 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=381 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp/C/39997ef645c4489c8dcd86a1b772f498 2024-11-23T16:13:59,885 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4550087eb34ce466d194f9ce96539302/B of 4550087eb34ce466d194f9ce96539302 into 7e2d25392d2147289fc05b625c6ffd7e(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T16:13:59,885 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4550087eb34ce466d194f9ce96539302: 2024-11-23T16:13:59,885 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., storeName=4550087eb34ce466d194f9ce96539302/B, priority=13, startTime=1732378439357; duration=0sec 2024-11-23T16:13:59,885 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T16:13:59,885 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4550087eb34ce466d194f9ce96539302:B 2024-11-23T16:13:59,890 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4550087eb34ce466d194f9ce96539302#C#compaction#74 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T16:13:59,891 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp/C/e9b592eaee564dd981371aa047057668 is 50, key is test_row_0/C:col10/1732378438763/Put/seqid=0 2024-11-23T16:13:59,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp/A/d0f5a1fdd6264a5db5af52ec7c477538 as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/A/d0f5a1fdd6264a5db5af52ec7c477538 2024-11-23T16:13:59,894 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:13:59,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40410 deadline: 1732378499888, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:13:59,895 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:59,895 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:59,895 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:13:59,898 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:13:59,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 190 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40416 deadline: 1732378499888, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:13:59,899 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:59,899 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:13:59,899 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:13:59,900 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/A/d0f5a1fdd6264a5db5af52ec7c477538, entries=150, sequenceid=381, filesize=12.0 K 2024-11-23T16:13:59,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp/B/99ff1a763d9d4a1db56eb883ff76d25c as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/B/99ff1a763d9d4a1db56eb883ff76d25c 2024-11-23T16:13:59,907 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/B/99ff1a763d9d4a1db56eb883ff76d25c, entries=150, sequenceid=381, filesize=12.0 K 2024-11-23T16:13:59,908 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073741911_1087 (size=13085) 2024-11-23T16:13:59,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp/C/39997ef645c4489c8dcd86a1b772f498 as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/C/39997ef645c4489c8dcd86a1b772f498 2024-11-23T16:13:59,916 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp/C/e9b592eaee564dd981371aa047057668 as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/C/e9b592eaee564dd981371aa047057668 2024-11-23T16:13:59,917 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/C/39997ef645c4489c8dcd86a1b772f498, entries=150, sequenceid=381, filesize=12.0 K 2024-11-23T16:13:59,920 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~365.02 KB/373776, currentSize=53.67 KB/54960 for 4550087eb34ce466d194f9ce96539302 in 561ms, sequenceid=381, compaction requested=false 2024-11-23T16:13:59,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2603): Flush status journal for 4550087eb34ce466d194f9ce96539302: 2024-11-23T16:13:59,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302. 2024-11-23T16:13:59,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=20 2024-11-23T16:13:59,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45799 {}] master.HMaster(4169): Remote procedure done, pid=20 2024-11-23T16:13:59,925 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=19 2024-11-23T16:13:59,925 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0320 sec 2024-11-23T16:13:59,927 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4550087eb34ce466d194f9ce96539302/C of 4550087eb34ce466d194f9ce96539302 into e9b592eaee564dd981371aa047057668(size=12.8 K), total size for store is 24.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T16:13:59,927 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4550087eb34ce466d194f9ce96539302: 2024-11-23T16:13:59,927 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., storeName=4550087eb34ce466d194f9ce96539302/C, priority=13, startTime=1732378439357; duration=0sec 2024-11-23T16:13:59,928 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T16:13:59,928 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4550087eb34ce466d194f9ce96539302:C 2024-11-23T16:13:59,930 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees in 1.0430 sec 2024-11-23T16:14:00,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-23T16:14:00,028 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-23T16:14:00,029 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45799 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-23T16:14:00,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45799 {}] procedure2.ProcedureExecutor(1139): Stored pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees 2024-11-23T16:14:00,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-23T16:14:00,032 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-23T16:14:00,033 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-23T16:14:00,033 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=22, ppid=21, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-23T16:14:00,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-23T16:14:00,186 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-23T16:14:00,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302. 2024-11-23T16:14:00,187 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2902): Flushing 4550087eb34ce466d194f9ce96539302 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-23T16:14:00,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4550087eb34ce466d194f9ce96539302, store=A 2024-11-23T16:14:00,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-23T16:14:00,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4550087eb34ce466d194f9ce96539302, store=B 2024-11-23T16:14:00,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-23T16:14:00,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4550087eb34ce466d194f9ce96539302, store=C 2024-11-23T16:14:00,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-23T16:14:00,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp/A/a54aaa1609e84a88abedfa731cdff996 is 50, key is test_row_0/A:col10/1732378439559/Put/seqid=0 2024-11-23T16:14:00,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(8855): Flush requested on 4550087eb34ce466d194f9ce96539302 2024-11-23T16:14:00,202 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302. as already flushing 2024-11-23T16:14:00,237 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073741912_1088 (size=12301) 2024-11-23T16:14:00,281 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:00,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 199 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40416 deadline: 1732378500272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:00,282 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1a7c5470b472:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732378440239, max timestamp=1732378440278Num uniques 859; 2024-11-23T16:14:00,282 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1a7c5470b472:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732378440239, max timestamp=1732378440278Num uniques 859; 2024-11-23T16:14:00,282 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1a7c5470b472:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-23T16:14:00,283 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1a7c5470b472:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-23T16:14:00,283 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1a7c5470b472:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732378440239, max timestamp=1732378440278Num uniques 859; 2024-11-23T16:14:00,283 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1a7c5470b472:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-23T16:14:00,284 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:00,284 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:00,284 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:14:00,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-23T16:14:00,391 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:00,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40410 deadline: 1732378500382, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:00,392 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:00,392 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:00,392 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:00,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 200 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40416 deadline: 1732378500388, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:00,392 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:14:00,393 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:00,393 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:00,393 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:14:00,502 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:00,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40410 deadline: 1732378500498, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:00,505 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:00,506 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:00,506 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:14:00,601 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:00,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 201 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40416 deadline: 1732378500598, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:00,602 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:00,602 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:00,602 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:14:00,638 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=395 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp/A/a54aaa1609e84a88abedfa731cdff996 2024-11-23T16:14:00,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp/B/72cb7d127e4f4684a103c0f1c839f476 is 50, key is test_row_0/B:col10/1732378439559/Put/seqid=0 2024-11-23T16:14:00,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-23T16:14:00,666 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073741913_1089 (size=12301) 2024-11-23T16:14:00,667 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=395 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp/B/72cb7d127e4f4684a103c0f1c839f476 2024-11-23T16:14:00,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp/C/5a58da32da9b4fdb90683ce3f3db6a2f is 50, key is test_row_0/C:col10/1732378439559/Put/seqid=0 2024-11-23T16:14:00,705 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073741914_1090 (size=12301) 2024-11-23T16:14:00,706 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=395 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp/C/5a58da32da9b4fdb90683ce3f3db6a2f 2024-11-23T16:14:00,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp/A/a54aaa1609e84a88abedfa731cdff996 as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/A/a54aaa1609e84a88abedfa731cdff996 2024-11-23T16:14:00,714 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:00,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40410 deadline: 1732378500708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:00,715 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:00,715 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:00,715 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:14:00,731 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/A/a54aaa1609e84a88abedfa731cdff996, entries=150, sequenceid=395, filesize=12.0 K 2024-11-23T16:14:00,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp/B/72cb7d127e4f4684a103c0f1c839f476 as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/B/72cb7d127e4f4684a103c0f1c839f476 2024-11-23T16:14:00,740 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/B/72cb7d127e4f4684a103c0f1c839f476, entries=150, sequenceid=395, filesize=12.0 K 2024-11-23T16:14:00,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp/C/5a58da32da9b4fdb90683ce3f3db6a2f as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/C/5a58da32da9b4fdb90683ce3f3db6a2f 2024-11-23T16:14:00,752 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/C/5a58da32da9b4fdb90683ce3f3db6a2f, entries=150, sequenceid=395, filesize=12.0 K 2024-11-23T16:14:00,756 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 4550087eb34ce466d194f9ce96539302 in 569ms, sequenceid=395, compaction requested=true 2024-11-23T16:14:00,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2603): Flush status journal for 4550087eb34ce466d194f9ce96539302: 2024-11-23T16:14:00,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302. 2024-11-23T16:14:00,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=22 2024-11-23T16:14:00,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45799 {}] master.HMaster(4169): Remote procedure done, pid=22 2024-11-23T16:14:00,760 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=22, resume processing ppid=21 2024-11-23T16:14:00,760 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 725 msec 2024-11-23T16:14:00,764 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees in 733 msec 2024-11-23T16:14:00,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(8855): Flush requested on 4550087eb34ce466d194f9ce96539302 2024-11-23T16:14:00,914 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4550087eb34ce466d194f9ce96539302 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-11-23T16:14:00,914 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4550087eb34ce466d194f9ce96539302, store=A 2024-11-23T16:14:00,914 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-23T16:14:00,915 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4550087eb34ce466d194f9ce96539302, store=B 2024-11-23T16:14:00,915 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-23T16:14:00,915 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4550087eb34ce466d194f9ce96539302, store=C 2024-11-23T16:14:00,915 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-23T16:14:00,927 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp/A/0c9edef6d991444aba04145dbe3a4370 is 50, key is test_row_0/A:col10/1732378440347/Put/seqid=0 2024-11-23T16:14:00,939 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073741915_1091 (size=12301) 2024-11-23T16:14:00,941 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=431 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp/A/0c9edef6d991444aba04145dbe3a4370 2024-11-23T16:14:00,967 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp/B/6c39d6556c2d48918c81287073117e6b is 50, key is test_row_0/B:col10/1732378440347/Put/seqid=0 2024-11-23T16:14:00,975 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:00,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 210 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40416 deadline: 1732378500973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:00,976 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:00,976 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:00,976 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:14:00,977 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073741916_1092 (size=12301) 2024-11-23T16:14:00,979 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=431 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp/B/6c39d6556c2d48918c81287073117e6b 2024-11-23T16:14:00,994 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp/C/cebc6cb5ab6449fdb5a5db4cd1fe73b7 is 50, key is test_row_0/C:col10/1732378440347/Put/seqid=0 2024-11-23T16:14:00,999 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073741917_1093 (size=12301) 2024-11-23T16:14:01,022 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:01,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40410 deadline: 1732378501018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:01,023 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:01,023 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:01,023 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:14:01,081 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:01,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 211 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40416 deadline: 1732378501077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:01,081 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:01,081 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:01,081 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:14:01,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-23T16:14:01,169 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-23T16:14:01,169 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45799 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-23T16:14:01,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45799 {}] procedure2.ProcedureExecutor(1139): Stored pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees 2024-11-23T16:14:01,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-23T16:14:01,171 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-23T16:14:01,172 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-23T16:14:01,172 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=24, ppid=23, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-23T16:14:01,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-23T16:14:01,291 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:01,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 212 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40416 deadline: 1732378501288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:01,292 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:01,292 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:01,292 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:14:01,325 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-23T16:14:01,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302. 2024-11-23T16:14:01,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302. as already flushing 2024-11-23T16:14:01,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302. 2024-11-23T16:14:01,326 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 4550087eb34ce466d194f9ce96539302, NAME => 'TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:14:01,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 4550087eb34ce466d194f9ce96539302, NAME => 'TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:14:01,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45799 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4550087eb34ce466d194f9ce96539302, NAME => 'TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4550087eb34ce466d194f9ce96539302, NAME => 'TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:14:01,400 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=431 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp/C/cebc6cb5ab6449fdb5a5db4cd1fe73b7 2024-11-23T16:14:01,408 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp/A/0c9edef6d991444aba04145dbe3a4370 as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/A/0c9edef6d991444aba04145dbe3a4370 2024-11-23T16:14:01,415 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/A/0c9edef6d991444aba04145dbe3a4370, entries=150, sequenceid=431, filesize=12.0 K 2024-11-23T16:14:01,417 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp/B/6c39d6556c2d48918c81287073117e6b as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/B/6c39d6556c2d48918c81287073117e6b 2024-11-23T16:14:01,428 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/B/6c39d6556c2d48918c81287073117e6b, entries=150, sequenceid=431, filesize=12.0 K 2024-11-23T16:14:01,429 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp/C/cebc6cb5ab6449fdb5a5db4cd1fe73b7 as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/C/cebc6cb5ab6449fdb5a5db4cd1fe73b7 2024-11-23T16:14:01,443 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/C/cebc6cb5ab6449fdb5a5db4cd1fe73b7, entries=150, sequenceid=431, filesize=12.0 K 2024-11-23T16:14:01,445 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=46.96 KB/48090 for 4550087eb34ce466d194f9ce96539302 in 530ms, sequenceid=431, compaction requested=true 2024-11-23T16:14:01,445 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4550087eb34ce466d194f9ce96539302: 2024-11-23T16:14:01,445 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-23T16:14:01,445 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4550087eb34ce466d194f9ce96539302:A, priority=-2147483648, current under compaction store size is 1 2024-11-23T16:14:01,445 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T16:14:01,446 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-23T16:14:01,446 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4550087eb34ce466d194f9ce96539302:B, priority=-2147483648, current under compaction store size is 2 2024-11-23T16:14:01,446 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T16:14:01,446 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4550087eb34ce466d194f9ce96539302:C, priority=-2147483648, current under compaction store size is 3 2024-11-23T16:14:01,446 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T16:14:01,448 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49988 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-23T16:14:01,448 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HStore(1541): 4550087eb34ce466d194f9ce96539302/A is initiating minor compaction (all files) 2024-11-23T16:14:01,448 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4550087eb34ce466d194f9ce96539302/A in TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302. 2024-11-23T16:14:01,448 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/A/01693751cc6c4dedbba2ca6e53da143a, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/A/d0f5a1fdd6264a5db5af52ec7c477538, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/A/a54aaa1609e84a88abedfa731cdff996, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/A/0c9edef6d991444aba04145dbe3a4370] into tmpdir=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp, totalSize=48.8 K 2024-11-23T16:14:01,449 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49988 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-23T16:14:01,449 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HStore(1541): 4550087eb34ce466d194f9ce96539302/B is initiating minor compaction (all files) 2024-11-23T16:14:01,449 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4550087eb34ce466d194f9ce96539302/B in TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302. 2024-11-23T16:14:01,449 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/B/7e2d25392d2147289fc05b625c6ffd7e, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/B/99ff1a763d9d4a1db56eb883ff76d25c, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/B/72cb7d127e4f4684a103c0f1c839f476, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/B/6c39d6556c2d48918c81287073117e6b] into tmpdir=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp, totalSize=48.8 K 2024-11-23T16:14:01,449 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.Compactor(225): Compacting 01693751cc6c4dedbba2ca6e53da143a, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=345, earliestPutTs=1732378436579 2024-11-23T16:14:01,450 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.Compactor(225): Compacting 7e2d25392d2147289fc05b625c6ffd7e, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=345, earliestPutTs=1732378436579 2024-11-23T16:14:01,450 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.Compactor(225): Compacting d0f5a1fdd6264a5db5af52ec7c477538, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=381, earliestPutTs=1732378438892 2024-11-23T16:14:01,451 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.Compactor(225): Compacting 99ff1a763d9d4a1db56eb883ff76d25c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=381, earliestPutTs=1732378438892 2024-11-23T16:14:01,451 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.Compactor(225): Compacting a54aaa1609e84a88abedfa731cdff996, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=395, earliestPutTs=1732378439553 2024-11-23T16:14:01,451 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.Compactor(225): Compacting 72cb7d127e4f4684a103c0f1c839f476, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=395, earliestPutTs=1732378439553 2024-11-23T16:14:01,451 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0c9edef6d991444aba04145dbe3a4370, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=431, earliestPutTs=1732378440347 2024-11-23T16:14:01,452 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.Compactor(225): Compacting 6c39d6556c2d48918c81287073117e6b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=431, earliestPutTs=1732378440347 2024-11-23T16:14:01,474 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4550087eb34ce466d194f9ce96539302#A#compaction#81 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T16:14:01,475 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp/A/931d2e23573241bf87f7ab69cb8eaad0 is 50, key is test_row_0/A:col10/1732378440347/Put/seqid=0 2024-11-23T16:14:01,476 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4550087eb34ce466d194f9ce96539302#B#compaction#82 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T16:14:01,476 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp/B/d09fddfcc6d342d28960ff46f1c4eccc is 50, key is test_row_0/B:col10/1732378440347/Put/seqid=0 2024-11-23T16:14:01,480 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-23T16:14:01,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302. 2024-11-23T16:14:01,481 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2902): Flushing 4550087eb34ce466d194f9ce96539302 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-23T16:14:01,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4550087eb34ce466d194f9ce96539302, store=A 2024-11-23T16:14:01,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-23T16:14:01,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4550087eb34ce466d194f9ce96539302, store=B 2024-11-23T16:14:01,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-23T16:14:01,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4550087eb34ce466d194f9ce96539302, store=C 2024-11-23T16:14:01,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-23T16:14:01,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-23T16:14:01,495 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073741918_1094 (size=13221) 2024-11-23T16:14:01,504 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp/A/931d2e23573241bf87f7ab69cb8eaad0 as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/A/931d2e23573241bf87f7ab69cb8eaad0 2024-11-23T16:14:01,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp/A/c4b235e35cf545daa9610d9c78d893cb is 50, key is test_row_0/A:col10/1732378440938/Put/seqid=0 2024-11-23T16:14:01,510 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073741919_1095 (size=13221) 2024-11-23T16:14:01,512 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 4550087eb34ce466d194f9ce96539302/A of 4550087eb34ce466d194f9ce96539302 into 931d2e23573241bf87f7ab69cb8eaad0(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T16:14:01,512 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4550087eb34ce466d194f9ce96539302: 2024-11-23T16:14:01,512 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., storeName=4550087eb34ce466d194f9ce96539302/A, priority=12, startTime=1732378441445; duration=0sec 2024-11-23T16:14:01,512 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T16:14:01,513 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4550087eb34ce466d194f9ce96539302:A 2024-11-23T16:14:01,513 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-23T16:14:01,515 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49988 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-23T16:14:01,515 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HStore(1541): 4550087eb34ce466d194f9ce96539302/C is initiating minor compaction (all files) 2024-11-23T16:14:01,515 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4550087eb34ce466d194f9ce96539302/C in TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302. 2024-11-23T16:14:01,515 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/C/e9b592eaee564dd981371aa047057668, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/C/39997ef645c4489c8dcd86a1b772f498, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/C/5a58da32da9b4fdb90683ce3f3db6a2f, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/C/cebc6cb5ab6449fdb5a5db4cd1fe73b7] into tmpdir=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp, totalSize=48.8 K 2024-11-23T16:14:01,515 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.Compactor(225): Compacting e9b592eaee564dd981371aa047057668, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=345, earliestPutTs=1732378436579 2024-11-23T16:14:01,516 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.Compactor(225): Compacting 39997ef645c4489c8dcd86a1b772f498, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=381, earliestPutTs=1732378438892 2024-11-23T16:14:01,516 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5a58da32da9b4fdb90683ce3f3db6a2f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=395, earliestPutTs=1732378439553 2024-11-23T16:14:01,517 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.Compactor(225): Compacting cebc6cb5ab6449fdb5a5db4cd1fe73b7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=431, earliestPutTs=1732378440347 2024-11-23T16:14:01,518 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073741920_1096 (size=12301) 2024-11-23T16:14:01,535 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4550087eb34ce466d194f9ce96539302#C#compaction#84 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T16:14:01,536 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp/C/3fa1ea9b2d9342b898e0e507574a73a9 is 50, key is test_row_0/C:col10/1732378440347/Put/seqid=0 2024-11-23T16:14:01,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(8855): Flush requested on 4550087eb34ce466d194f9ce96539302 2024-11-23T16:14:01,541 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302. as already flushing 2024-11-23T16:14:01,557 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073741921_1097 (size=13221) 2024-11-23T16:14:01,566 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp/C/3fa1ea9b2d9342b898e0e507574a73a9 as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/C/3fa1ea9b2d9342b898e0e507574a73a9 2024-11-23T16:14:01,575 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 4550087eb34ce466d194f9ce96539302/C of 4550087eb34ce466d194f9ce96539302 into 3fa1ea9b2d9342b898e0e507574a73a9(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T16:14:01,575 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4550087eb34ce466d194f9ce96539302: 2024-11-23T16:14:01,575 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., storeName=4550087eb34ce466d194f9ce96539302/C, priority=12, startTime=1732378441446; duration=0sec 2024-11-23T16:14:01,576 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T16:14:01,576 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4550087eb34ce466d194f9ce96539302:C 2024-11-23T16:14:01,624 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1a7c5470b472:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732378441599, max timestamp=1732378441620Num uniques 859; 2024-11-23T16:14:01,624 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1a7c5470b472:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-23T16:14:01,627 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1a7c5470b472:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732378441599, max timestamp=1732378441620Num uniques 859; 2024-11-23T16:14:01,627 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1a7c5470b472:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-23T16:14:01,629 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1a7c5470b472:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732378441609, max timestamp=1732378441620Num uniques 859; 2024-11-23T16:14:01,630 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1a7c5470b472:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-23T16:14:01,696 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:01,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 201 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40410 deadline: 1732378501689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:01,697 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:01,697 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:01,697 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:01,697 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:14:01,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 225 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40416 deadline: 1732378501689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:01,697 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:01,698 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:01,698 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:14:01,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-23T16:14:01,814 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:01,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 202 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40410 deadline: 1732378501813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:01,815 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:01,815 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:01,815 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:01,815 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:14:01,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 226 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40416 deadline: 1732378501814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:01,815 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:01,816 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:01,816 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:14:01,920 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=441 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp/A/c4b235e35cf545daa9610d9c78d893cb 2024-11-23T16:14:01,922 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp/B/d09fddfcc6d342d28960ff46f1c4eccc as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/B/d09fddfcc6d342d28960ff46f1c4eccc 2024-11-23T16:14:01,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp/B/de0389d71ef04f33973daf9c07a28f26 is 50, key is test_row_0/B:col10/1732378440938/Put/seqid=0 2024-11-23T16:14:01,949 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 4550087eb34ce466d194f9ce96539302/B of 4550087eb34ce466d194f9ce96539302 into d09fddfcc6d342d28960ff46f1c4eccc(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T16:14:01,949 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4550087eb34ce466d194f9ce96539302: 2024-11-23T16:14:01,949 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., storeName=4550087eb34ce466d194f9ce96539302/B, priority=12, startTime=1732378441445; duration=0sec 2024-11-23T16:14:01,949 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T16:14:01,949 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4550087eb34ce466d194f9ce96539302:B 2024-11-23T16:14:01,966 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073741922_1098 (size=12301) 2024-11-23T16:14:01,968 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=441 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp/B/de0389d71ef04f33973daf9c07a28f26 2024-11-23T16:14:01,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp/C/98ecc8312acd4a6280968eefea7966ea is 50, key is test_row_0/C:col10/1732378440938/Put/seqid=0 2024-11-23T16:14:01,999 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073741923_1099 (size=12301) 2024-11-23T16:14:02,032 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:02,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 227 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40416 deadline: 1732378502028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:02,033 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:02,033 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:02,033 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:14:02,037 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:02,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 203 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40410 deadline: 1732378502029, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:02,037 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:02,038 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:02,038 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:14:02,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-23T16:14:02,343 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:02,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 228 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40416 deadline: 1732378502339, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:02,344 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:02,345 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:02,345 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:14:02,349 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:02,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 204 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40410 deadline: 1732378502348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:02,350 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:02,350 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:02,350 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:14:02,401 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=441 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp/C/98ecc8312acd4a6280968eefea7966ea 2024-11-23T16:14:02,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp/A/c4b235e35cf545daa9610d9c78d893cb as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/A/c4b235e35cf545daa9610d9c78d893cb 2024-11-23T16:14:02,417 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/A/c4b235e35cf545daa9610d9c78d893cb, entries=150, sequenceid=441, filesize=12.0 K 2024-11-23T16:14:02,419 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp/B/de0389d71ef04f33973daf9c07a28f26 as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/B/de0389d71ef04f33973daf9c07a28f26 2024-11-23T16:14:02,426 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/B/de0389d71ef04f33973daf9c07a28f26, entries=150, sequenceid=441, filesize=12.0 K 2024-11-23T16:14:02,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp/C/98ecc8312acd4a6280968eefea7966ea as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/C/98ecc8312acd4a6280968eefea7966ea 2024-11-23T16:14:02,435 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/C/98ecc8312acd4a6280968eefea7966ea, entries=150, sequenceid=441, filesize=12.0 K 2024-11-23T16:14:02,437 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=136.42 KB/139690 for 4550087eb34ce466d194f9ce96539302 in 956ms, sequenceid=441, compaction requested=false 2024-11-23T16:14:02,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2603): Flush status journal for 4550087eb34ce466d194f9ce96539302: 2024-11-23T16:14:02,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302. 2024-11-23T16:14:02,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=24 2024-11-23T16:14:02,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45799 {}] master.HMaster(4169): Remote procedure done, pid=24 2024-11-23T16:14:02,442 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=24, resume processing ppid=23 2024-11-23T16:14:02,442 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2670 sec 2024-11-23T16:14:02,444 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees in 1.2730 sec 2024-11-23T16:14:02,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(8855): Flush requested on 4550087eb34ce466d194f9ce96539302 2024-11-23T16:14:02,862 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4550087eb34ce466d194f9ce96539302 3/3 column families, dataSize=143.13 KB heapSize=405.30 KB 2024-11-23T16:14:02,862 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4550087eb34ce466d194f9ce96539302, store=A 2024-11-23T16:14:02,862 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-23T16:14:02,862 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4550087eb34ce466d194f9ce96539302, store=B 2024-11-23T16:14:02,862 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-23T16:14:02,862 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4550087eb34ce466d194f9ce96539302, store=C 2024-11-23T16:14:02,862 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-23T16:14:02,877 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp/A/29f09049b1a44163bf6468a5a6702a8a is 50, key is test_row_0/A:col10/1732378441686/Put/seqid=0 2024-11-23T16:14:02,893 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:02,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 208 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40410 deadline: 1732378502883, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:02,894 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:02,894 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:02,894 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:02,894 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:14:02,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 233 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40416 deadline: 1732378502889, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:02,894 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:02,894 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:02,894 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:14:02,907 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073741924_1100 (size=14741) 2024-11-23T16:14:03,002 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:03,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 209 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40410 deadline: 1732378502998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:03,003 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:03,003 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:03,003 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:03,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 234 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40416 deadline: 1732378502998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:03,003 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:14:03,006 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:03,006 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:03,006 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:14:03,211 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:03,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 210 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40410 deadline: 1732378503208, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:03,213 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:03,213 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:03,213 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:14:03,221 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:03,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 235 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40416 deadline: 1732378503218, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:03,222 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:03,222 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:03,222 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:14:03,308 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=481 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp/A/29f09049b1a44163bf6468a5a6702a8a 2024-11-23T16:14:03,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-23T16:14:03,318 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-23T16:14:03,319 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45799 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-23T16:14:03,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45799 {}] procedure2.ProcedureExecutor(1139): Stored pid=25, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees 2024-11-23T16:14:03,320 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp/B/f156f8df24a24cbaae55775e160f4e4d is 50, key is test_row_0/B:col10/1732378441686/Put/seqid=0 2024-11-23T16:14:03,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-23T16:14:03,322 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=25, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-23T16:14:03,323 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=25, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-23T16:14:03,323 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=26, ppid=25, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-23T16:14:03,342 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073741925_1101 (size=12301) 2024-11-23T16:14:03,343 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=481 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp/B/f156f8df24a24cbaae55775e160f4e4d 2024-11-23T16:14:03,359 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp/C/161c7cc7bd7849a186a599868d6362eb is 50, key is test_row_0/C:col10/1732378441686/Put/seqid=0 2024-11-23T16:14:03,365 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073741926_1102 (size=12301) 2024-11-23T16:14:03,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-23T16:14:03,478 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-23T16:14:03,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302. 2024-11-23T16:14:03,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302. as already flushing 2024-11-23T16:14:03,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302. 2024-11-23T16:14:03,479 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => 4550087eb34ce466d194f9ce96539302, NAME => 'TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:14:03,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => 4550087eb34ce466d194f9ce96539302, NAME => 'TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:14:03,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45799 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4550087eb34ce466d194f9ce96539302, NAME => 'TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4550087eb34ce466d194f9ce96539302, NAME => 'TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:14:03,520 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:03,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 211 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40410 deadline: 1732378503519, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:03,521 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:03,521 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:03,521 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:14:03,533 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:03,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 236 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40416 deadline: 1732378503528, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:03,535 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:03,535 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:03,535 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:14:03,633 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-23T16:14:03,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302. 2024-11-23T16:14:03,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302. as already flushing 2024-11-23T16:14:03,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302. 2024-11-23T16:14:03,633 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => 4550087eb34ce466d194f9ce96539302, NAME => 'TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:14:03,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => 4550087eb34ce466d194f9ce96539302, NAME => 'TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:14:03,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45799 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4550087eb34ce466d194f9ce96539302, NAME => 'TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4550087eb34ce466d194f9ce96539302, NAME => 'TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:14:03,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-23T16:14:03,766 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=481 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp/C/161c7cc7bd7849a186a599868d6362eb 2024-11-23T16:14:03,773 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp/A/29f09049b1a44163bf6468a5a6702a8a as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/A/29f09049b1a44163bf6468a5a6702a8a 2024-11-23T16:14:03,778 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/A/29f09049b1a44163bf6468a5a6702a8a, entries=200, sequenceid=481, filesize=14.4 K 2024-11-23T16:14:03,781 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp/B/f156f8df24a24cbaae55775e160f4e4d as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/B/f156f8df24a24cbaae55775e160f4e4d 2024-11-23T16:14:03,786 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-23T16:14:03,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302. 2024-11-23T16:14:03,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302. as already flushing 2024-11-23T16:14:03,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302. 2024-11-23T16:14:03,786 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => 4550087eb34ce466d194f9ce96539302, NAME => 'TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:14:03,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => 4550087eb34ce466d194f9ce96539302, NAME => 'TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:14:03,787 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/B/f156f8df24a24cbaae55775e160f4e4d, entries=150, sequenceid=481, filesize=12.0 K 2024-11-23T16:14:03,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45799 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4550087eb34ce466d194f9ce96539302, NAME => 'TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4550087eb34ce466d194f9ce96539302, NAME => 'TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:14:03,800 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp/C/161c7cc7bd7849a186a599868d6362eb as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/C/161c7cc7bd7849a186a599868d6362eb 2024-11-23T16:14:03,810 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/C/161c7cc7bd7849a186a599868d6362eb, entries=150, sequenceid=481, filesize=12.0 K 2024-11-23T16:14:03,811 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~143.13 KB/146560, heapSize ~405.25 KB/414976, currentSize=40.25 KB/41220 for 4550087eb34ce466d194f9ce96539302 in 949ms, sequenceid=481, compaction requested=true 2024-11-23T16:14:03,811 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4550087eb34ce466d194f9ce96539302: 2024-11-23T16:14:03,811 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4550087eb34ce466d194f9ce96539302:A, priority=-2147483648, current under compaction store size is 1 2024-11-23T16:14:03,812 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T16:14:03,812 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T16:14:03,812 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T16:14:03,812 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4550087eb34ce466d194f9ce96539302:B, priority=-2147483648, current under compaction store size is 2 2024-11-23T16:14:03,812 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T16:14:03,812 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4550087eb34ce466d194f9ce96539302:C, priority=-2147483648, current under compaction store size is 3 2024-11-23T16:14:03,812 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T16:14:03,813 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37823 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T16:14:03,813 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HStore(1541): 4550087eb34ce466d194f9ce96539302/B is initiating minor compaction (all files) 2024-11-23T16:14:03,813 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4550087eb34ce466d194f9ce96539302/B in TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302. 2024-11-23T16:14:03,813 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40263 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T16:14:03,814 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HStore(1541): 4550087eb34ce466d194f9ce96539302/A is initiating minor compaction (all files) 2024-11-23T16:14:03,814 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/B/d09fddfcc6d342d28960ff46f1c4eccc, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/B/de0389d71ef04f33973daf9c07a28f26, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/B/f156f8df24a24cbaae55775e160f4e4d] into tmpdir=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp, totalSize=36.9 K 2024-11-23T16:14:03,814 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4550087eb34ce466d194f9ce96539302/A in TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302. 2024-11-23T16:14:03,814 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/A/931d2e23573241bf87f7ab69cb8eaad0, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/A/c4b235e35cf545daa9610d9c78d893cb, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/A/29f09049b1a44163bf6468a5a6702a8a] into tmpdir=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp, totalSize=39.3 K 2024-11-23T16:14:03,814 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.Compactor(225): Compacting d09fddfcc6d342d28960ff46f1c4eccc, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=431, earliestPutTs=1732378440347 2024-11-23T16:14:03,814 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.Compactor(225): Compacting 931d2e23573241bf87f7ab69cb8eaad0, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=431, earliestPutTs=1732378440347 2024-11-23T16:14:03,815 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.Compactor(225): Compacting de0389d71ef04f33973daf9c07a28f26, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=441, earliestPutTs=1732378440938 2024-11-23T16:14:03,815 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.Compactor(225): Compacting c4b235e35cf545daa9610d9c78d893cb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=441, earliestPutTs=1732378440938 2024-11-23T16:14:03,815 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.Compactor(225): Compacting f156f8df24a24cbaae55775e160f4e4d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=481, earliestPutTs=1732378441676 2024-11-23T16:14:03,815 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.Compactor(225): Compacting 29f09049b1a44163bf6468a5a6702a8a, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=481, earliestPutTs=1732378441668 2024-11-23T16:14:03,826 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4550087eb34ce466d194f9ce96539302#B#compaction#90 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-23T16:14:03,826 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp/B/a7eef1e90284423c941b9567eef3a9b5 is 50, key is test_row_0/B:col10/1732378441686/Put/seqid=0 2024-11-23T16:14:03,828 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4550087eb34ce466d194f9ce96539302#A#compaction#91 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T16:14:03,829 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp/A/d4d1987cf9b04fc195900f036d91ad7d is 50, key is test_row_0/A:col10/1732378441686/Put/seqid=0 2024-11-23T16:14:03,873 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073741928_1104 (size=13323) 2024-11-23T16:14:03,874 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073741927_1103 (size=13323) 2024-11-23T16:14:03,940 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-23T16:14:03,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302. 2024-11-23T16:14:03,941 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2902): Flushing 4550087eb34ce466d194f9ce96539302 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-11-23T16:14:03,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4550087eb34ce466d194f9ce96539302, store=A 2024-11-23T16:14:03,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-23T16:14:03,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4550087eb34ce466d194f9ce96539302, store=B 2024-11-23T16:14:03,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-23T16:14:03,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4550087eb34ce466d194f9ce96539302, store=C 2024-11-23T16:14:03,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-23T16:14:03,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-23T16:14:03,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp/A/3c1395862a224d8dbd8ad2c3819db16e is 50, key is test_row_0/A:col10/1732378442886/Put/seqid=0 2024-11-23T16:14:03,966 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073741929_1105 (size=12301) 2024-11-23T16:14:04,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(8855): Flush requested on 4550087eb34ce466d194f9ce96539302 2024-11-23T16:14:04,030 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302. as already flushing 2024-11-23T16:14:04,105 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1a7c5470b472:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1732378444048, max timestamp=1732378444102Num uniques 859; 2024-11-23T16:14:04,105 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1a7c5470b472:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1732378444048, max timestamp=1732378444102Num uniques 859; 2024-11-23T16:14:04,106 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1a7c5470b472:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-23T16:14:04,106 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1a7c5470b472:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-23T16:14:04,108 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1a7c5470b472:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732378444088, max timestamp=1732378444102Num uniques 859; 2024-11-23T16:14:04,108 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1a7c5470b472:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-23T16:14:04,183 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:04,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 231 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40410 deadline: 1732378504176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:04,184 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:04,184 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:04,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 251 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40416 deadline: 1732378504179, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:04,184 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:04,184 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:14:04,184 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:04,184 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:04,185 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:14:04,284 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp/A/d4d1987cf9b04fc195900f036d91ad7d as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/A/d4d1987cf9b04fc195900f036d91ad7d 2024-11-23T16:14:04,287 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp/B/a7eef1e90284423c941b9567eef3a9b5 as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/B/a7eef1e90284423c941b9567eef3a9b5 2024-11-23T16:14:04,289 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:04,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 232 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40410 deadline: 1732378504289, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:04,290 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:04,290 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:04,290 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:04,290 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:14:04,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 252 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40416 deadline: 1732378504289, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:04,291 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:04,291 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:04,291 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:14:04,298 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4550087eb34ce466d194f9ce96539302/B of 4550087eb34ce466d194f9ce96539302 into a7eef1e90284423c941b9567eef3a9b5(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T16:14:04,298 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4550087eb34ce466d194f9ce96539302: 2024-11-23T16:14:04,298 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., storeName=4550087eb34ce466d194f9ce96539302/B, priority=13, startTime=1732378443812; duration=0sec 2024-11-23T16:14:04,299 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T16:14:04,299 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4550087eb34ce466d194f9ce96539302:B 2024-11-23T16:14:04,299 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T16:14:04,299 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4550087eb34ce466d194f9ce96539302/A of 4550087eb34ce466d194f9ce96539302 into d4d1987cf9b04fc195900f036d91ad7d(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T16:14:04,299 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4550087eb34ce466d194f9ce96539302: 2024-11-23T16:14:04,299 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., storeName=4550087eb34ce466d194f9ce96539302/A, priority=13, startTime=1732378443811; duration=0sec 2024-11-23T16:14:04,299 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T16:14:04,299 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4550087eb34ce466d194f9ce96539302:A 2024-11-23T16:14:04,300 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37823 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T16:14:04,300 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HStore(1541): 4550087eb34ce466d194f9ce96539302/C is initiating minor compaction (all files) 2024-11-23T16:14:04,300 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4550087eb34ce466d194f9ce96539302/C in TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302. 2024-11-23T16:14:04,300 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/C/3fa1ea9b2d9342b898e0e507574a73a9, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/C/98ecc8312acd4a6280968eefea7966ea, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/C/161c7cc7bd7849a186a599868d6362eb] into tmpdir=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp, totalSize=36.9 K 2024-11-23T16:14:04,301 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.Compactor(225): Compacting 3fa1ea9b2d9342b898e0e507574a73a9, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=431, earliestPutTs=1732378440347 2024-11-23T16:14:04,301 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.Compactor(225): Compacting 98ecc8312acd4a6280968eefea7966ea, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=441, earliestPutTs=1732378440938 2024-11-23T16:14:04,302 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.Compactor(225): Compacting 161c7cc7bd7849a186a599868d6362eb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=481, earliestPutTs=1732378441676 2024-11-23T16:14:04,314 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4550087eb34ce466d194f9ce96539302#C#compaction#93 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T16:14:04,315 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp/C/7abd61d73ed44ac798a81202518d8dd8 is 50, key is test_row_0/C:col10/1732378441686/Put/seqid=0 2024-11-23T16:14:04,324 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073741930_1106 (size=13323) 2024-11-23T16:14:04,333 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp/C/7abd61d73ed44ac798a81202518d8dd8 as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/C/7abd61d73ed44ac798a81202518d8dd8 2024-11-23T16:14:04,340 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4550087eb34ce466d194f9ce96539302/C of 4550087eb34ce466d194f9ce96539302 into 7abd61d73ed44ac798a81202518d8dd8(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T16:14:04,340 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4550087eb34ce466d194f9ce96539302: 2024-11-23T16:14:04,340 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., storeName=4550087eb34ce466d194f9ce96539302/C, priority=13, startTime=1732378443812; duration=0sec 2024-11-23T16:14:04,341 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T16:14:04,341 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4550087eb34ce466d194f9ce96539302:C 2024-11-23T16:14:04,368 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=490 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp/A/3c1395862a224d8dbd8ad2c3819db16e 2024-11-23T16:14:04,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp/B/3dc994c368194a0aae01340624b1b2c1 is 50, key is test_row_0/B:col10/1732378442886/Put/seqid=0 2024-11-23T16:14:04,386 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073741931_1107 (size=12301) 2024-11-23T16:14:04,388 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=490 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp/B/3dc994c368194a0aae01340624b1b2c1 2024-11-23T16:14:04,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp/C/b7af0fa5777b415486f6c1b77a4bd80f is 50, key is test_row_0/C:col10/1732378442886/Put/seqid=0 2024-11-23T16:14:04,411 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073741932_1108 (size=12301) 2024-11-23T16:14:04,411 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=490 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp/C/b7af0fa5777b415486f6c1b77a4bd80f 2024-11-23T16:14:04,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp/A/3c1395862a224d8dbd8ad2c3819db16e as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/A/3c1395862a224d8dbd8ad2c3819db16e 2024-11-23T16:14:04,435 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/A/3c1395862a224d8dbd8ad2c3819db16e, entries=150, sequenceid=490, filesize=12.0 K 2024-11-23T16:14:04,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp/B/3dc994c368194a0aae01340624b1b2c1 as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/B/3dc994c368194a0aae01340624b1b2c1 2024-11-23T16:14:04,442 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/B/3dc994c368194a0aae01340624b1b2c1, entries=150, sequenceid=490, filesize=12.0 K 2024-11-23T16:14:04,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp/C/b7af0fa5777b415486f6c1b77a4bd80f as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/C/b7af0fa5777b415486f6c1b77a4bd80f 2024-11-23T16:14:04,454 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/C/b7af0fa5777b415486f6c1b77a4bd80f, entries=150, sequenceid=490, filesize=12.0 K 2024-11-23T16:14:04,455 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=140.89 KB/144270 for 4550087eb34ce466d194f9ce96539302 in 513ms, sequenceid=490, compaction requested=false 2024-11-23T16:14:04,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2603): Flush status journal for 4550087eb34ce466d194f9ce96539302: 2024-11-23T16:14:04,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302. 2024-11-23T16:14:04,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=26 2024-11-23T16:14:04,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45799 {}] master.HMaster(4169): Remote procedure done, pid=26 2024-11-23T16:14:04,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-23T16:14:04,460 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=26, resume processing ppid=25 2024-11-23T16:14:04,460 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=25, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1340 sec 2024-11-23T16:14:04,463 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees in 1.1420 sec 2024-11-23T16:14:04,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(8855): Flush requested on 4550087eb34ce466d194f9ce96539302 2024-11-23T16:14:04,500 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4550087eb34ce466d194f9ce96539302 3/3 column families, dataSize=147.60 KB heapSize=415.45 KB 2024-11-23T16:14:04,501 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4550087eb34ce466d194f9ce96539302, store=A 2024-11-23T16:14:04,501 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-23T16:14:04,501 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4550087eb34ce466d194f9ce96539302, store=B 2024-11-23T16:14:04,501 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-23T16:14:04,501 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4550087eb34ce466d194f9ce96539302, store=C 2024-11-23T16:14:04,501 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-23T16:14:04,508 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp/A/bbe5675fdd884b74965e68499839f715 is 50, key is test_row_0/A:col10/1732378444178/Put/seqid=0 2024-11-23T16:14:04,515 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:04,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 236 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40410 deadline: 1732378504512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:04,516 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:04,516 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:04,516 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:04,516 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:14:04,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 256 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40416 deadline: 1732378504515, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:04,517 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:04,517 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:04,517 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:14:04,530 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073741933_1109 (size=14741) 2024-11-23T16:14:04,622 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:04,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 237 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40410 deadline: 1732378504618, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:04,623 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:04,623 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:04,623 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:14:04,634 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:04,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 257 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40416 deadline: 1732378504629, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:04,635 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:04,635 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:04,635 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:14:04,835 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:04,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 238 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40410 deadline: 1732378504828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:04,835 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:04,835 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:04,835 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:14:04,842 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:04,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 258 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40416 deadline: 1732378504838, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:04,843 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:04,843 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:04,843 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:14:04,932 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=530 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp/A/bbe5675fdd884b74965e68499839f715 2024-11-23T16:14:04,950 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp/B/8a959f6ef6414a94b18a4c6915150e9d is 50, key is test_row_0/B:col10/1732378444178/Put/seqid=0 2024-11-23T16:14:04,986 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073741934_1110 (size=12301) 2024-11-23T16:14:04,987 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=530 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp/B/8a959f6ef6414a94b18a4c6915150e9d 2024-11-23T16:14:04,991 INFO [Thread-156 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-156. 2024-11-23T16:14:04,991 DEBUG [Thread-156 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-23T16:14:04,992 INFO [Thread-160 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-160. 2024-11-23T16:14:04,992 DEBUG [Thread-160 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-23T16:14:04,995 INFO [Thread-154 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-154. 2024-11-23T16:14:04,995 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp/C/33f6b41fe17c4db0859f0a55a753a1d3 is 50, key is test_row_0/C:col10/1732378444178/Put/seqid=0 2024-11-23T16:14:04,995 DEBUG [Thread-154 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-23T16:14:04,997 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T16:14:04,997 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T16:14:04,997 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T16:14:04,997 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T16:14:04,997 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-23T16:14:04,997 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T16:14:04,998 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-23T16:14:04,998 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T16:14:04,998 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-23T16:14:05,000 INFO [Thread-158 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-158. 2024-11-23T16:14:05,000 DEBUG [Thread-158 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-23T16:14:05,000 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T16:14:05,001 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T16:14:05,001 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-23T16:14:05,014 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073741935_1111 (size=12301) 2024-11-23T16:14:05,149 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:05,149 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:05,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 259 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40416 deadline: 1732378505148, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:05,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 239 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40410 deadline: 1732378505148, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:05,150 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:05,150 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:05,150 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:05,150 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:05,150 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:14:05,150 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:14:05,415 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=530 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp/C/33f6b41fe17c4db0859f0a55a753a1d3 2024-11-23T16:14:05,422 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp/A/bbe5675fdd884b74965e68499839f715 as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/A/bbe5675fdd884b74965e68499839f715 2024-11-23T16:14:05,427 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/A/bbe5675fdd884b74965e68499839f715, entries=200, sequenceid=530, filesize=14.4 K 2024-11-23T16:14:05,428 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp/B/8a959f6ef6414a94b18a4c6915150e9d as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/B/8a959f6ef6414a94b18a4c6915150e9d 2024-11-23T16:14:05,434 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/B/8a959f6ef6414a94b18a4c6915150e9d, entries=150, sequenceid=530, filesize=12.0 K 2024-11-23T16:14:05,435 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp/C/33f6b41fe17c4db0859f0a55a753a1d3 as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/C/33f6b41fe17c4db0859f0a55a753a1d3 2024-11-23T16:14:05,440 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/C/33f6b41fe17c4db0859f0a55a753a1d3, entries=150, sequenceid=530, filesize=12.0 K 2024-11-23T16:14:05,441 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~415.41 KB/425376, currentSize=33.54 KB/34350 for 4550087eb34ce466d194f9ce96539302 in 941ms, sequenceid=530, compaction requested=true 2024-11-23T16:14:05,441 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4550087eb34ce466d194f9ce96539302: 2024-11-23T16:14:05,441 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4550087eb34ce466d194f9ce96539302:A, priority=-2147483648, current under compaction store size is 1 2024-11-23T16:14:05,441 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T16:14:05,441 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T16:14:05,441 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4550087eb34ce466d194f9ce96539302:B, priority=-2147483648, current under compaction store size is 2 2024-11-23T16:14:05,441 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T16:14:05,441 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T16:14:05,441 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4550087eb34ce466d194f9ce96539302:C, priority=-2147483648, current under compaction store size is 3 2024-11-23T16:14:05,441 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T16:14:05,442 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40365 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T16:14:05,443 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37925 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T16:14:05,443 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HStore(1541): 4550087eb34ce466d194f9ce96539302/A is initiating minor compaction (all files) 2024-11-23T16:14:05,443 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HStore(1541): 4550087eb34ce466d194f9ce96539302/B is initiating minor compaction (all files) 2024-11-23T16:14:05,443 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4550087eb34ce466d194f9ce96539302/B in TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302. 2024-11-23T16:14:05,443 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4550087eb34ce466d194f9ce96539302/A in TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302. 2024-11-23T16:14:05,443 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/B/a7eef1e90284423c941b9567eef3a9b5, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/B/3dc994c368194a0aae01340624b1b2c1, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/B/8a959f6ef6414a94b18a4c6915150e9d] into tmpdir=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp, totalSize=37.0 K 2024-11-23T16:14:05,443 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/A/d4d1987cf9b04fc195900f036d91ad7d, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/A/3c1395862a224d8dbd8ad2c3819db16e, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/A/bbe5675fdd884b74965e68499839f715] into tmpdir=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp, totalSize=39.4 K 2024-11-23T16:14:05,443 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.Compactor(225): Compacting a7eef1e90284423c941b9567eef3a9b5, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=481, earliestPutTs=1732378441676 2024-11-23T16:14:05,443 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.Compactor(225): Compacting d4d1987cf9b04fc195900f036d91ad7d, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=481, earliestPutTs=1732378441676 2024-11-23T16:14:05,444 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.Compactor(225): Compacting 3dc994c368194a0aae01340624b1b2c1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=490, earliestPutTs=1732378442863 2024-11-23T16:14:05,444 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3c1395862a224d8dbd8ad2c3819db16e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=490, earliestPutTs=1732378442863 2024-11-23T16:14:05,444 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.Compactor(225): Compacting 8a959f6ef6414a94b18a4c6915150e9d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=530, earliestPutTs=1732378444166 2024-11-23T16:14:05,444 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.Compactor(225): Compacting bbe5675fdd884b74965e68499839f715, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=530, earliestPutTs=1732378444166 2024-11-23T16:14:05,454 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4550087eb34ce466d194f9ce96539302#A#compaction#99 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T16:14:05,455 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp/A/881aed0576c94eadb532eb3d6fae4a85 is 50, key is test_row_0/A:col10/1732378444178/Put/seqid=0 2024-11-23T16:14:05,458 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4550087eb34ce466d194f9ce96539302#B#compaction#100 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T16:14:05,458 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp/B/d37ad426d879496e872fc19a0058feb0 is 50, key is test_row_0/B:col10/1732378444178/Put/seqid=0 2024-11-23T16:14:05,465 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073741936_1112 (size=13425) 2024-11-23T16:14:05,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-23T16:14:05,468 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-23T16:14:05,471 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp/A/881aed0576c94eadb532eb3d6fae4a85 as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/A/881aed0576c94eadb532eb3d6fae4a85 2024-11-23T16:14:05,473 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073741937_1113 (size=13425) 2024-11-23T16:14:05,476 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4550087eb34ce466d194f9ce96539302/A of 4550087eb34ce466d194f9ce96539302 into 881aed0576c94eadb532eb3d6fae4a85(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T16:14:05,476 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4550087eb34ce466d194f9ce96539302: 2024-11-23T16:14:05,476 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., storeName=4550087eb34ce466d194f9ce96539302/A, priority=13, startTime=1732378445441; duration=0sec 2024-11-23T16:14:05,477 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T16:14:05,477 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4550087eb34ce466d194f9ce96539302:A 2024-11-23T16:14:05,477 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T16:14:05,478 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37925 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T16:14:05,478 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HStore(1541): 4550087eb34ce466d194f9ce96539302/C is initiating minor compaction (all files) 2024-11-23T16:14:05,478 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4550087eb34ce466d194f9ce96539302/C in TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302. 2024-11-23T16:14:05,478 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/C/7abd61d73ed44ac798a81202518d8dd8, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/C/b7af0fa5777b415486f6c1b77a4bd80f, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/C/33f6b41fe17c4db0859f0a55a753a1d3] into tmpdir=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp, totalSize=37.0 K 2024-11-23T16:14:05,479 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7abd61d73ed44ac798a81202518d8dd8, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=481, earliestPutTs=1732378441676 2024-11-23T16:14:05,479 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.Compactor(225): Compacting b7af0fa5777b415486f6c1b77a4bd80f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=490, earliestPutTs=1732378442863 2024-11-23T16:14:05,480 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.Compactor(225): Compacting 33f6b41fe17c4db0859f0a55a753a1d3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=530, earliestPutTs=1732378444166 2024-11-23T16:14:05,487 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4550087eb34ce466d194f9ce96539302#C#compaction#101 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T16:14:05,488 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp/C/d729f51fbd5545b8882012637ad66fb3 is 50, key is test_row_0/C:col10/1732378444178/Put/seqid=0 2024-11-23T16:14:05,495 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073741938_1114 (size=13425) 2024-11-23T16:14:05,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(8855): Flush requested on 4550087eb34ce466d194f9ce96539302 2024-11-23T16:14:05,661 INFO [Thread-147 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-147. 2024-11-23T16:14:05,661 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4550087eb34ce466d194f9ce96539302 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-23T16:14:05,661 DEBUG [Thread-147 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-23T16:14:05,661 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T16:14:05,662 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T16:14:05,662 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-23T16:14:05,662 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4550087eb34ce466d194f9ce96539302, store=A 2024-11-23T16:14:05,662 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-23T16:14:05,662 INFO [Thread-151 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-151. 2024-11-23T16:14:05,662 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4550087eb34ce466d194f9ce96539302, store=B 2024-11-23T16:14:05,662 DEBUG [Thread-151 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-23T16:14:05,662 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-23T16:14:05,662 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T16:14:05,662 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4550087eb34ce466d194f9ce96539302, store=C 2024-11-23T16:14:05,662 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-23T16:14:05,662 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T16:14:05,662 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-23T16:14:05,669 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp/A/17a58f50a89e4c1b83f14910b610c13c is 50, key is test_row_0/A:col10/1732378444510/Put/seqid=0 2024-11-23T16:14:05,674 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073741939_1115 (size=12301) 2024-11-23T16:14:05,676 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=541 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp/A/17a58f50a89e4c1b83f14910b610c13c 2024-11-23T16:14:05,687 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp/B/f6628fef4f684bc78e2252b54d5065b5 is 50, key is test_row_0/B:col10/1732378444510/Put/seqid=0 2024-11-23T16:14:05,694 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073741940_1116 (size=12301) 2024-11-23T16:14:05,880 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp/B/d37ad426d879496e872fc19a0058feb0 as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/B/d37ad426d879496e872fc19a0058feb0 2024-11-23T16:14:05,888 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4550087eb34ce466d194f9ce96539302/B of 4550087eb34ce466d194f9ce96539302 into d37ad426d879496e872fc19a0058feb0(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T16:14:05,888 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4550087eb34ce466d194f9ce96539302: 2024-11-23T16:14:05,888 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., storeName=4550087eb34ce466d194f9ce96539302/B, priority=13, startTime=1732378445441; duration=0sec 2024-11-23T16:14:05,888 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T16:14:05,888 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4550087eb34ce466d194f9ce96539302:B 2024-11-23T16:14:05,902 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp/C/d729f51fbd5545b8882012637ad66fb3 as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/C/d729f51fbd5545b8882012637ad66fb3 2024-11-23T16:14:05,909 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4550087eb34ce466d194f9ce96539302/C of 4550087eb34ce466d194f9ce96539302 into d729f51fbd5545b8882012637ad66fb3(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T16:14:05,910 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4550087eb34ce466d194f9ce96539302: 2024-11-23T16:14:05,910 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., storeName=4550087eb34ce466d194f9ce96539302/C, priority=13, startTime=1732378445441; duration=0sec 2024-11-23T16:14:05,910 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T16:14:05,910 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4550087eb34ce466d194f9ce96539302:C 2024-11-23T16:14:06,095 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=541 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp/B/f6628fef4f684bc78e2252b54d5065b5 2024-11-23T16:14:06,104 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp/C/4fcda3d692f1404c819e60acfb0064cd is 50, key is test_row_0/C:col10/1732378444510/Put/seqid=0 2024-11-23T16:14:06,111 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073741941_1117 (size=12301) 2024-11-23T16:14:06,512 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=541 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp/C/4fcda3d692f1404c819e60acfb0064cd 2024-11-23T16:14:06,518 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp/A/17a58f50a89e4c1b83f14910b610c13c as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/A/17a58f50a89e4c1b83f14910b610c13c 2024-11-23T16:14:06,522 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/A/17a58f50a89e4c1b83f14910b610c13c, entries=150, sequenceid=541, filesize=12.0 K 2024-11-23T16:14:06,523 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp/B/f6628fef4f684bc78e2252b54d5065b5 as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/B/f6628fef4f684bc78e2252b54d5065b5 2024-11-23T16:14:06,529 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/B/f6628fef4f684bc78e2252b54d5065b5, entries=150, sequenceid=541, filesize=12.0 K 2024-11-23T16:14:06,529 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp/C/4fcda3d692f1404c819e60acfb0064cd as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/C/4fcda3d692f1404c819e60acfb0064cd 2024-11-23T16:14:06,534 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/C/4fcda3d692f1404c819e60acfb0064cd, entries=150, sequenceid=541, filesize=12.0 K 2024-11-23T16:14:06,535 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=0 B/0 for 4550087eb34ce466d194f9ce96539302 in 874ms, sequenceid=541, compaction requested=false 2024-11-23T16:14:06,535 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4550087eb34ce466d194f9ce96539302: 2024-11-23T16:14:07,599 INFO [Thread-145 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-145. 2024-11-23T16:14:07,600 DEBUG [Thread-145 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-23T16:14:07,600 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T16:14:07,600 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T16:14:07,600 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-23T16:14:07,610 INFO [Thread-143 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-143. 2024-11-23T16:14:07,610 DEBUG [Thread-143 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-23T16:14:07,610 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T16:14:07,610 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T16:14:07,610 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-23T16:14:07,630 INFO [Thread-149 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-149. 2024-11-23T16:14:07,630 DEBUG [Thread-149 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-23T16:14:07,630 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T16:14:07,631 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T16:14:07,631 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-23T16:14:07,631 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-23T16:14:07,631 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 30 2024-11-23T16:14:07,631 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 25 2024-11-23T16:14:07,631 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 172 2024-11-23T16:14:07,631 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 26 2024-11-23T16:14:07,631 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 187 2024-11-23T16:14:07,631 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-23T16:14:07,632 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2494 2024-11-23T16:14:07,632 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2587 2024-11-23T16:14:07,632 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-23T16:14:07,632 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1184 2024-11-23T16:14:07,632 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3552 rows 2024-11-23T16:14:07,632 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1175 2024-11-23T16:14:07,632 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3525 rows 2024-11-23T16:14:07,632 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-23T16:14:07,632 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMixedAtomicity(AcidGuaranteesTestBase.java:117) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-23T16:14:07,632 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T16:14:07,632 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T16:14:07,632 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-23T16:14:07,637 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45799 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-23T16:14:07,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45799 {}] procedure2.ProcedureExecutor(1139): Stored pid=27, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-23T16:14:07,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-23T16:14:07,646 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732378447646"}]},"ts":"1732378447646"} 2024-11-23T16:14:07,648 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-23T16:14:07,648 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-23T16:14:07,650 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=28, ppid=27, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-23T16:14:07,655 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=4550087eb34ce466d194f9ce96539302, UNASSIGN}] 2024-11-23T16:14:07,656 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=4550087eb34ce466d194f9ce96539302, UNASSIGN 2024-11-23T16:14:07,657 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=4550087eb34ce466d194f9ce96539302, regionState=CLOSING, regionLocation=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:07,659 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=4550087eb34ce466d194f9ce96539302, UNASSIGN because future has completed 2024-11-23T16:14:07,660 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-23T16:14:07,660 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=30, ppid=29, state=RUNNABLE, hasLock=false; CloseRegionProcedure 4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375}] 2024-11-23T16:14:07,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-23T16:14:07,817 INFO [RS_CLOSE_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] handler.UnassignRegionHandler(122): Close 4550087eb34ce466d194f9ce96539302 2024-11-23T16:14:07,817 DEBUG [RS_CLOSE_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-23T16:14:07,817 DEBUG [RS_CLOSE_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1722): Closing 4550087eb34ce466d194f9ce96539302, disabling compactions & flushes 2024-11-23T16:14:07,817 INFO [RS_CLOSE_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302. 2024-11-23T16:14:07,818 DEBUG [RS_CLOSE_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302. 2024-11-23T16:14:07,818 DEBUG [RS_CLOSE_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302. after waiting 0 ms 2024-11-23T16:14:07,818 DEBUG [RS_CLOSE_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302. 2024-11-23T16:14:07,818 INFO [RS_CLOSE_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(2902): Flushing 4550087eb34ce466d194f9ce96539302 3/3 column families, dataSize=20.13 KB heapSize=60.52 KB 2024-11-23T16:14:07,818 DEBUG [RS_CLOSE_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4550087eb34ce466d194f9ce96539302, store=A 2024-11-23T16:14:07,818 DEBUG [RS_CLOSE_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-23T16:14:07,818 DEBUG [RS_CLOSE_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4550087eb34ce466d194f9ce96539302, store=B 2024-11-23T16:14:07,818 DEBUG [RS_CLOSE_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-23T16:14:07,818 DEBUG [RS_CLOSE_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4550087eb34ce466d194f9ce96539302, store=C 2024-11-23T16:14:07,818 DEBUG [RS_CLOSE_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-23T16:14:07,823 DEBUG [RS_CLOSE_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp/A/14345d6e6d604c3e91a9c231b145d6dc is 50, key is test_row_0/A:col10/1732378447608/Put/seqid=0 2024-11-23T16:14:07,827 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073741942_1118 (size=12301) 2024-11-23T16:14:07,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-23T16:14:08,228 INFO [RS_CLOSE_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=549 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp/A/14345d6e6d604c3e91a9c231b145d6dc 2024-11-23T16:14:08,236 DEBUG [RS_CLOSE_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp/B/235f6ae49a8a47a6a350d9a4aacbcabc is 50, key is test_row_0/B:col10/1732378447608/Put/seqid=0 2024-11-23T16:14:08,240 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073741943_1119 (size=12301) 2024-11-23T16:14:08,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-23T16:14:08,630 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-23T16:14:08,641 INFO [RS_CLOSE_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=549 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp/B/235f6ae49a8a47a6a350d9a4aacbcabc 2024-11-23T16:14:08,648 DEBUG [RS_CLOSE_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp/C/0876305e3b094ae5aedaf6d6927354d9 is 50, key is test_row_0/C:col10/1732378447608/Put/seqid=0 2024-11-23T16:14:08,651 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073741944_1120 (size=12301) 2024-11-23T16:14:08,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-23T16:14:09,052 INFO [RS_CLOSE_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=549 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp/C/0876305e3b094ae5aedaf6d6927354d9 2024-11-23T16:14:09,058 DEBUG [RS_CLOSE_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp/A/14345d6e6d604c3e91a9c231b145d6dc as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/A/14345d6e6d604c3e91a9c231b145d6dc 2024-11-23T16:14:09,061 INFO [RS_CLOSE_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/A/14345d6e6d604c3e91a9c231b145d6dc, entries=150, sequenceid=549, filesize=12.0 K 2024-11-23T16:14:09,062 DEBUG [RS_CLOSE_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp/B/235f6ae49a8a47a6a350d9a4aacbcabc as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/B/235f6ae49a8a47a6a350d9a4aacbcabc 2024-11-23T16:14:09,067 INFO [RS_CLOSE_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/B/235f6ae49a8a47a6a350d9a4aacbcabc, entries=150, sequenceid=549, filesize=12.0 K 2024-11-23T16:14:09,068 DEBUG [RS_CLOSE_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/.tmp/C/0876305e3b094ae5aedaf6d6927354d9 as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/C/0876305e3b094ae5aedaf6d6927354d9 2024-11-23T16:14:09,072 INFO [RS_CLOSE_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/C/0876305e3b094ae5aedaf6d6927354d9, entries=150, sequenceid=549, filesize=12.0 K 2024-11-23T16:14:09,073 INFO [RS_CLOSE_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(3140): Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=0 B/0 for 4550087eb34ce466d194f9ce96539302 in 1255ms, sequenceid=549, compaction requested=true 2024-11-23T16:14:09,074 DEBUG [StoreCloser-TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/A/11d8100dbeac4ff3b5ef044fae4ffa8c, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/A/78d5b357a7364ecfab262ea4b8c8dbb8, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/A/f65beed7b7e145bf9b59699554bd853f, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/A/ee9e8b462a504fcf91d03bf8e1d210ca, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/A/fc702aca2401483fb69f73fc0cc7511e, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/A/75298ec17fac429488835c77a3eadcc7, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/A/81b73bdad0f243829088dad2f30d5500, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/A/6144aee27cd5447484ee86e1ade2e898, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/A/499bc90d1c6243ad83bb04294c762777, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/A/24c91d4415f741bca4e2ee0a4646c557, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/A/4af5583f912e4fceae6174b1df9bc52c, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/A/6928613c49f8437aa7f059a0eff46d3d, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/A/6caa843f43804b5b838656e781bfd55a, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/A/a19ce2b875ff463bae8bf01b447e2b3d, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/A/4f136979e1eb43d1a21d5f90e040a57a, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/A/7f44ac7f49c542b1bf09ac3260d5a75c, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/A/7b655553f25d453683d972d6ddb07d66, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/A/98da1ac6baa647e2b68406fa88e32244, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/A/22598940ec8d4369a016eef57a7b26bd, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/A/361f8b365cd04c05af07ccabf074cbb8, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/A/0abf882ec4b54304a78ed6a8c8005e4a, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/A/c8d72f03685c415a8dbc665ce04aa59e, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/A/38c4d67e5e68480b9a4b0a689a6ad87a, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/A/01693751cc6c4dedbba2ca6e53da143a, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/A/d0f5a1fdd6264a5db5af52ec7c477538, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/A/a54aaa1609e84a88abedfa731cdff996, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/A/931d2e23573241bf87f7ab69cb8eaad0, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/A/0c9edef6d991444aba04145dbe3a4370, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/A/c4b235e35cf545daa9610d9c78d893cb, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/A/29f09049b1a44163bf6468a5a6702a8a, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/A/d4d1987cf9b04fc195900f036d91ad7d, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/A/3c1395862a224d8dbd8ad2c3819db16e, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/A/bbe5675fdd884b74965e68499839f715] to archive 2024-11-23T16:14:09,077 DEBUG [StoreCloser-TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-23T16:14:09,082 DEBUG [StoreCloser-TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/A/11d8100dbeac4ff3b5ef044fae4ffa8c to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/A/11d8100dbeac4ff3b5ef044fae4ffa8c 2024-11-23T16:14:09,083 DEBUG [StoreCloser-TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/A/78d5b357a7364ecfab262ea4b8c8dbb8 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/A/78d5b357a7364ecfab262ea4b8c8dbb8 2024-11-23T16:14:09,084 DEBUG [StoreCloser-TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/A/f65beed7b7e145bf9b59699554bd853f to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/A/f65beed7b7e145bf9b59699554bd853f 2024-11-23T16:14:09,085 DEBUG [StoreCloser-TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/A/ee9e8b462a504fcf91d03bf8e1d210ca to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/A/ee9e8b462a504fcf91d03bf8e1d210ca 2024-11-23T16:14:09,086 DEBUG [StoreCloser-TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/A/fc702aca2401483fb69f73fc0cc7511e to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/A/fc702aca2401483fb69f73fc0cc7511e 2024-11-23T16:14:09,088 DEBUG [StoreCloser-TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/A/75298ec17fac429488835c77a3eadcc7 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/A/75298ec17fac429488835c77a3eadcc7 2024-11-23T16:14:09,089 DEBUG [StoreCloser-TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/A/81b73bdad0f243829088dad2f30d5500 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/A/81b73bdad0f243829088dad2f30d5500 2024-11-23T16:14:09,090 DEBUG [StoreCloser-TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/A/6144aee27cd5447484ee86e1ade2e898 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/A/6144aee27cd5447484ee86e1ade2e898 2024-11-23T16:14:09,091 DEBUG [StoreCloser-TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/A/499bc90d1c6243ad83bb04294c762777 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/A/499bc90d1c6243ad83bb04294c762777 2024-11-23T16:14:09,092 DEBUG [StoreCloser-TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/A/24c91d4415f741bca4e2ee0a4646c557 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/A/24c91d4415f741bca4e2ee0a4646c557 2024-11-23T16:14:09,093 DEBUG [StoreCloser-TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/A/4af5583f912e4fceae6174b1df9bc52c to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/A/4af5583f912e4fceae6174b1df9bc52c 2024-11-23T16:14:09,095 DEBUG [StoreCloser-TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/A/6928613c49f8437aa7f059a0eff46d3d to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/A/6928613c49f8437aa7f059a0eff46d3d 2024-11-23T16:14:09,096 DEBUG [StoreCloser-TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/A/6caa843f43804b5b838656e781bfd55a to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/A/6caa843f43804b5b838656e781bfd55a 2024-11-23T16:14:09,097 DEBUG [StoreCloser-TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/A/a19ce2b875ff463bae8bf01b447e2b3d to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/A/a19ce2b875ff463bae8bf01b447e2b3d 2024-11-23T16:14:09,098 DEBUG [StoreCloser-TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/A/4f136979e1eb43d1a21d5f90e040a57a to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/A/4f136979e1eb43d1a21d5f90e040a57a 2024-11-23T16:14:09,099 DEBUG [StoreCloser-TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/A/7f44ac7f49c542b1bf09ac3260d5a75c to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/A/7f44ac7f49c542b1bf09ac3260d5a75c 2024-11-23T16:14:09,100 DEBUG [StoreCloser-TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/A/7b655553f25d453683d972d6ddb07d66 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/A/7b655553f25d453683d972d6ddb07d66 2024-11-23T16:14:09,102 DEBUG [StoreCloser-TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/A/98da1ac6baa647e2b68406fa88e32244 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/A/98da1ac6baa647e2b68406fa88e32244 2024-11-23T16:14:09,103 DEBUG [StoreCloser-TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/A/22598940ec8d4369a016eef57a7b26bd to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/A/22598940ec8d4369a016eef57a7b26bd 2024-11-23T16:14:09,104 DEBUG [StoreCloser-TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/A/361f8b365cd04c05af07ccabf074cbb8 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/A/361f8b365cd04c05af07ccabf074cbb8 2024-11-23T16:14:09,105 DEBUG [StoreCloser-TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/A/0abf882ec4b54304a78ed6a8c8005e4a to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/A/0abf882ec4b54304a78ed6a8c8005e4a 2024-11-23T16:14:09,106 DEBUG [StoreCloser-TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/A/c8d72f03685c415a8dbc665ce04aa59e to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/A/c8d72f03685c415a8dbc665ce04aa59e 2024-11-23T16:14:09,108 DEBUG [StoreCloser-TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/A/38c4d67e5e68480b9a4b0a689a6ad87a to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/A/38c4d67e5e68480b9a4b0a689a6ad87a 2024-11-23T16:14:09,109 DEBUG [StoreCloser-TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/A/01693751cc6c4dedbba2ca6e53da143a to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/A/01693751cc6c4dedbba2ca6e53da143a 2024-11-23T16:14:09,110 DEBUG [StoreCloser-TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/A/d0f5a1fdd6264a5db5af52ec7c477538 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/A/d0f5a1fdd6264a5db5af52ec7c477538 2024-11-23T16:14:09,111 DEBUG [StoreCloser-TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/A/a54aaa1609e84a88abedfa731cdff996 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/A/a54aaa1609e84a88abedfa731cdff996 2024-11-23T16:14:09,112 DEBUG [StoreCloser-TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/A/931d2e23573241bf87f7ab69cb8eaad0 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/A/931d2e23573241bf87f7ab69cb8eaad0 2024-11-23T16:14:09,113 DEBUG [StoreCloser-TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/A/0c9edef6d991444aba04145dbe3a4370 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/A/0c9edef6d991444aba04145dbe3a4370 2024-11-23T16:14:09,114 DEBUG [StoreCloser-TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/A/c4b235e35cf545daa9610d9c78d893cb to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/A/c4b235e35cf545daa9610d9c78d893cb 2024-11-23T16:14:09,115 DEBUG [StoreCloser-TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/A/29f09049b1a44163bf6468a5a6702a8a to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/A/29f09049b1a44163bf6468a5a6702a8a 2024-11-23T16:14:09,116 DEBUG [StoreCloser-TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/A/d4d1987cf9b04fc195900f036d91ad7d to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/A/d4d1987cf9b04fc195900f036d91ad7d 2024-11-23T16:14:09,118 DEBUG [StoreCloser-TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/A/3c1395862a224d8dbd8ad2c3819db16e to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/A/3c1395862a224d8dbd8ad2c3819db16e 2024-11-23T16:14:09,119 DEBUG [StoreCloser-TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/A/bbe5675fdd884b74965e68499839f715 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/A/bbe5675fdd884b74965e68499839f715 2024-11-23T16:14:09,133 DEBUG [StoreCloser-TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/B/3a20a59de3934a8084afbc6412ec85d2, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/B/94094598c73f4c51bb19675082146305, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/B/db978ec1534a4f21bbb8f82b9e8aecdb, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/B/d356752a25ff4bb58f1ef25e56c4ab3b, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/B/5b8fa1919b424c4b8c61a999f7b59903, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/B/707e1bd9e5814838a99c40af9f3f7468, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/B/f6e8502fcd7142d8a289d3ed4d4efc7c, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/B/882cb90110d44b59ae01b0192184f7f5, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/B/99b189b77f60437d8b8abf639d661951, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/B/fb82bfda637d4937a56d7ceecd6d635d, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/B/9dc39ed6774a49ca89d6ff86db2c56e2, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/B/7c46d8a4b3b74a68befd5b9212928de6, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/B/92d606bbe87a4d2ca12c45d4f9e9bd80, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/B/02e054afe23a43ba83be6f0c5a06abbe, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/B/4bb8355660b944d6ad482d53cb9d269f, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/B/2f5bc73e426a4ab4835bb733a76492e9, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/B/0bec775274dd4ebea8ea426b7454137e, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/B/237cc2230d8347239f11d37f868210a8, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/B/498aadee42ea493db429f2f27dad6dca, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/B/08b3dc087e664d13b2c7cae2252b1ddf, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/B/dd0035fc11d141708484369b5f066f29, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/B/a831559cdf8d48b8888617f1ea7fccbf, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/B/7e2d25392d2147289fc05b625c6ffd7e, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/B/d5b0a45ddba84a27bb69b39717d3cef5, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/B/99ff1a763d9d4a1db56eb883ff76d25c, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/B/72cb7d127e4f4684a103c0f1c839f476, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/B/d09fddfcc6d342d28960ff46f1c4eccc, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/B/6c39d6556c2d48918c81287073117e6b, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/B/de0389d71ef04f33973daf9c07a28f26, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/B/a7eef1e90284423c941b9567eef3a9b5, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/B/f156f8df24a24cbaae55775e160f4e4d, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/B/3dc994c368194a0aae01340624b1b2c1, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/B/8a959f6ef6414a94b18a4c6915150e9d] to archive 2024-11-23T16:14:09,134 DEBUG [StoreCloser-TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-23T16:14:09,135 DEBUG [StoreCloser-TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/B/3a20a59de3934a8084afbc6412ec85d2 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/B/3a20a59de3934a8084afbc6412ec85d2 2024-11-23T16:14:09,137 DEBUG [StoreCloser-TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/B/94094598c73f4c51bb19675082146305 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/B/94094598c73f4c51bb19675082146305 2024-11-23T16:14:09,138 DEBUG [StoreCloser-TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/B/db978ec1534a4f21bbb8f82b9e8aecdb to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/B/db978ec1534a4f21bbb8f82b9e8aecdb 2024-11-23T16:14:09,139 DEBUG [StoreCloser-TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/B/d356752a25ff4bb58f1ef25e56c4ab3b to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/B/d356752a25ff4bb58f1ef25e56c4ab3b 2024-11-23T16:14:09,140 DEBUG [StoreCloser-TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/B/5b8fa1919b424c4b8c61a999f7b59903 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/B/5b8fa1919b424c4b8c61a999f7b59903 2024-11-23T16:14:09,141 DEBUG [StoreCloser-TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/B/707e1bd9e5814838a99c40af9f3f7468 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/B/707e1bd9e5814838a99c40af9f3f7468 2024-11-23T16:14:09,142 DEBUG [StoreCloser-TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/B/f6e8502fcd7142d8a289d3ed4d4efc7c to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/B/f6e8502fcd7142d8a289d3ed4d4efc7c 2024-11-23T16:14:09,143 DEBUG [StoreCloser-TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/B/882cb90110d44b59ae01b0192184f7f5 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/B/882cb90110d44b59ae01b0192184f7f5 2024-11-23T16:14:09,144 DEBUG [StoreCloser-TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/B/99b189b77f60437d8b8abf639d661951 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/B/99b189b77f60437d8b8abf639d661951 2024-11-23T16:14:09,145 DEBUG [StoreCloser-TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/B/fb82bfda637d4937a56d7ceecd6d635d to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/B/fb82bfda637d4937a56d7ceecd6d635d 2024-11-23T16:14:09,146 DEBUG [StoreCloser-TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/B/9dc39ed6774a49ca89d6ff86db2c56e2 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/B/9dc39ed6774a49ca89d6ff86db2c56e2 2024-11-23T16:14:09,147 DEBUG [StoreCloser-TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/B/7c46d8a4b3b74a68befd5b9212928de6 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/B/7c46d8a4b3b74a68befd5b9212928de6 2024-11-23T16:14:09,147 DEBUG [StoreCloser-TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/B/92d606bbe87a4d2ca12c45d4f9e9bd80 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/B/92d606bbe87a4d2ca12c45d4f9e9bd80 2024-11-23T16:14:09,149 DEBUG [StoreCloser-TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/B/02e054afe23a43ba83be6f0c5a06abbe to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/B/02e054afe23a43ba83be6f0c5a06abbe 2024-11-23T16:14:09,150 DEBUG [StoreCloser-TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/B/4bb8355660b944d6ad482d53cb9d269f to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/B/4bb8355660b944d6ad482d53cb9d269f 2024-11-23T16:14:09,151 DEBUG [StoreCloser-TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/B/2f5bc73e426a4ab4835bb733a76492e9 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/B/2f5bc73e426a4ab4835bb733a76492e9 2024-11-23T16:14:09,152 DEBUG [StoreCloser-TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/B/0bec775274dd4ebea8ea426b7454137e to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/B/0bec775274dd4ebea8ea426b7454137e 2024-11-23T16:14:09,153 DEBUG [StoreCloser-TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/B/237cc2230d8347239f11d37f868210a8 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/B/237cc2230d8347239f11d37f868210a8 2024-11-23T16:14:09,154 DEBUG [StoreCloser-TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/B/498aadee42ea493db429f2f27dad6dca to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/B/498aadee42ea493db429f2f27dad6dca 2024-11-23T16:14:09,155 DEBUG [StoreCloser-TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/B/08b3dc087e664d13b2c7cae2252b1ddf to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/B/08b3dc087e664d13b2c7cae2252b1ddf 2024-11-23T16:14:09,156 DEBUG [StoreCloser-TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/B/dd0035fc11d141708484369b5f066f29 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/B/dd0035fc11d141708484369b5f066f29 2024-11-23T16:14:09,157 DEBUG [StoreCloser-TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/B/a831559cdf8d48b8888617f1ea7fccbf to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/B/a831559cdf8d48b8888617f1ea7fccbf 2024-11-23T16:14:09,159 DEBUG [StoreCloser-TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/B/7e2d25392d2147289fc05b625c6ffd7e to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/B/7e2d25392d2147289fc05b625c6ffd7e 2024-11-23T16:14:09,160 DEBUG [StoreCloser-TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/B/d5b0a45ddba84a27bb69b39717d3cef5 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/B/d5b0a45ddba84a27bb69b39717d3cef5 2024-11-23T16:14:09,161 DEBUG [StoreCloser-TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/B/99ff1a763d9d4a1db56eb883ff76d25c to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/B/99ff1a763d9d4a1db56eb883ff76d25c 2024-11-23T16:14:09,162 DEBUG [StoreCloser-TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/B/72cb7d127e4f4684a103c0f1c839f476 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/B/72cb7d127e4f4684a103c0f1c839f476 2024-11-23T16:14:09,163 DEBUG [StoreCloser-TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/B/d09fddfcc6d342d28960ff46f1c4eccc to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/B/d09fddfcc6d342d28960ff46f1c4eccc 2024-11-23T16:14:09,164 DEBUG [StoreCloser-TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/B/6c39d6556c2d48918c81287073117e6b to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/B/6c39d6556c2d48918c81287073117e6b 2024-11-23T16:14:09,165 DEBUG [StoreCloser-TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/B/de0389d71ef04f33973daf9c07a28f26 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/B/de0389d71ef04f33973daf9c07a28f26 2024-11-23T16:14:09,166 DEBUG [StoreCloser-TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/B/a7eef1e90284423c941b9567eef3a9b5 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/B/a7eef1e90284423c941b9567eef3a9b5 2024-11-23T16:14:09,167 DEBUG [StoreCloser-TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/B/f156f8df24a24cbaae55775e160f4e4d to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/B/f156f8df24a24cbaae55775e160f4e4d 2024-11-23T16:14:09,168 DEBUG [StoreCloser-TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/B/3dc994c368194a0aae01340624b1b2c1 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/B/3dc994c368194a0aae01340624b1b2c1 2024-11-23T16:14:09,169 DEBUG [StoreCloser-TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/B/8a959f6ef6414a94b18a4c6915150e9d to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/B/8a959f6ef6414a94b18a4c6915150e9d 2024-11-23T16:14:09,171 DEBUG [StoreCloser-TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/C/1883edb4efeb459d8bff1bd5d8269417, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/C/fba9de58dfa84fe389f0869ffb0b8a5e, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/C/85bd609f56dd4d4891aaba05ecafd851, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/C/c82cabbae3434cd99b6938c95500884b, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/C/b0310f2db52d4173a306340c0da14e5c, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/C/4d8a2528d7474e3585dd0e27ab5d5867, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/C/380abb77007c4eafa3e6f8f635767cd0, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/C/e7c83b74b9fb4c8f91f5bb8778cd67c2, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/C/2bbcb47f9bc5473b9f50c28c5f1a1409, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/C/6b7d6b2c297a44d69495b87450b88e76, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/C/35709892451242dcae270339ab4be382, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/C/a0d5a3ced13040c89b6187089f1c5f4e, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/C/37dc0b6979544dcb873b925da413dc17, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/C/1646c45305c240aebf7c4ed037f78e08, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/C/22c4912c178e436d89d44351fd1301d5, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/C/9d94f4df87b142cd9d4b3fc9571fe4d0, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/C/1843f289f90e4db693c0c6bc0387dcf5, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/C/b9fd72e4d8d742839a4e929b4e5943d3, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/C/79ac225079bc4bd6bb395773461f3f42, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/C/85fcbe1353a1428dbf47aaf4aad9b51b, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/C/ebd861dd1bc440c4a7f62684ebd02003, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/C/3a9cdbf40e28420f929e60a9ed7f3463, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/C/e9b592eaee564dd981371aa047057668, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/C/57bab8993a1d45d6be4ba7f27b12e58e, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/C/39997ef645c4489c8dcd86a1b772f498, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/C/5a58da32da9b4fdb90683ce3f3db6a2f, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/C/3fa1ea9b2d9342b898e0e507574a73a9, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/C/cebc6cb5ab6449fdb5a5db4cd1fe73b7, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/C/98ecc8312acd4a6280968eefea7966ea, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/C/7abd61d73ed44ac798a81202518d8dd8, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/C/161c7cc7bd7849a186a599868d6362eb, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/C/b7af0fa5777b415486f6c1b77a4bd80f, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/C/33f6b41fe17c4db0859f0a55a753a1d3] to archive 2024-11-23T16:14:09,172 DEBUG [StoreCloser-TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-23T16:14:09,174 DEBUG [StoreCloser-TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/C/1883edb4efeb459d8bff1bd5d8269417 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/C/1883edb4efeb459d8bff1bd5d8269417 2024-11-23T16:14:09,175 DEBUG [StoreCloser-TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/C/fba9de58dfa84fe389f0869ffb0b8a5e to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/C/fba9de58dfa84fe389f0869ffb0b8a5e 2024-11-23T16:14:09,176 DEBUG [StoreCloser-TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/C/85bd609f56dd4d4891aaba05ecafd851 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/C/85bd609f56dd4d4891aaba05ecafd851 2024-11-23T16:14:09,177 DEBUG [StoreCloser-TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/C/c82cabbae3434cd99b6938c95500884b to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/C/c82cabbae3434cd99b6938c95500884b 2024-11-23T16:14:09,178 DEBUG [StoreCloser-TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/C/b0310f2db52d4173a306340c0da14e5c to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/C/b0310f2db52d4173a306340c0da14e5c 2024-11-23T16:14:09,179 DEBUG [StoreCloser-TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/C/4d8a2528d7474e3585dd0e27ab5d5867 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/C/4d8a2528d7474e3585dd0e27ab5d5867 2024-11-23T16:14:09,180 DEBUG [StoreCloser-TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/C/380abb77007c4eafa3e6f8f635767cd0 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/C/380abb77007c4eafa3e6f8f635767cd0 2024-11-23T16:14:09,181 DEBUG [StoreCloser-TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/C/e7c83b74b9fb4c8f91f5bb8778cd67c2 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/C/e7c83b74b9fb4c8f91f5bb8778cd67c2 2024-11-23T16:14:09,182 DEBUG [StoreCloser-TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/C/2bbcb47f9bc5473b9f50c28c5f1a1409 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/C/2bbcb47f9bc5473b9f50c28c5f1a1409 2024-11-23T16:14:09,183 DEBUG [StoreCloser-TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/C/6b7d6b2c297a44d69495b87450b88e76 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/C/6b7d6b2c297a44d69495b87450b88e76 2024-11-23T16:14:09,184 DEBUG [StoreCloser-TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/C/35709892451242dcae270339ab4be382 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/C/35709892451242dcae270339ab4be382 2024-11-23T16:14:09,185 DEBUG [StoreCloser-TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/C/a0d5a3ced13040c89b6187089f1c5f4e to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/C/a0d5a3ced13040c89b6187089f1c5f4e 2024-11-23T16:14:09,186 DEBUG [StoreCloser-TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/C/37dc0b6979544dcb873b925da413dc17 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/C/37dc0b6979544dcb873b925da413dc17 2024-11-23T16:14:09,187 DEBUG [StoreCloser-TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/C/1646c45305c240aebf7c4ed037f78e08 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/C/1646c45305c240aebf7c4ed037f78e08 2024-11-23T16:14:09,188 DEBUG [StoreCloser-TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/C/22c4912c178e436d89d44351fd1301d5 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/C/22c4912c178e436d89d44351fd1301d5 2024-11-23T16:14:09,189 DEBUG [StoreCloser-TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/C/9d94f4df87b142cd9d4b3fc9571fe4d0 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/C/9d94f4df87b142cd9d4b3fc9571fe4d0 2024-11-23T16:14:09,190 DEBUG [StoreCloser-TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/C/1843f289f90e4db693c0c6bc0387dcf5 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/C/1843f289f90e4db693c0c6bc0387dcf5 2024-11-23T16:14:09,191 DEBUG [StoreCloser-TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/C/b9fd72e4d8d742839a4e929b4e5943d3 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/C/b9fd72e4d8d742839a4e929b4e5943d3 2024-11-23T16:14:09,192 DEBUG [StoreCloser-TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/C/79ac225079bc4bd6bb395773461f3f42 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/C/79ac225079bc4bd6bb395773461f3f42 2024-11-23T16:14:09,194 DEBUG [StoreCloser-TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/C/85fcbe1353a1428dbf47aaf4aad9b51b to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/C/85fcbe1353a1428dbf47aaf4aad9b51b 2024-11-23T16:14:09,195 DEBUG [StoreCloser-TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/C/ebd861dd1bc440c4a7f62684ebd02003 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/C/ebd861dd1bc440c4a7f62684ebd02003 2024-11-23T16:14:09,196 DEBUG [StoreCloser-TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/C/3a9cdbf40e28420f929e60a9ed7f3463 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/C/3a9cdbf40e28420f929e60a9ed7f3463 2024-11-23T16:14:09,197 DEBUG [StoreCloser-TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/C/e9b592eaee564dd981371aa047057668 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/C/e9b592eaee564dd981371aa047057668 2024-11-23T16:14:09,198 DEBUG [StoreCloser-TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/C/57bab8993a1d45d6be4ba7f27b12e58e to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/C/57bab8993a1d45d6be4ba7f27b12e58e 2024-11-23T16:14:09,199 DEBUG [StoreCloser-TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/C/39997ef645c4489c8dcd86a1b772f498 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/C/39997ef645c4489c8dcd86a1b772f498 2024-11-23T16:14:09,200 DEBUG [StoreCloser-TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/C/5a58da32da9b4fdb90683ce3f3db6a2f to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/C/5a58da32da9b4fdb90683ce3f3db6a2f 2024-11-23T16:14:09,201 DEBUG [StoreCloser-TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/C/3fa1ea9b2d9342b898e0e507574a73a9 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/C/3fa1ea9b2d9342b898e0e507574a73a9 2024-11-23T16:14:09,203 DEBUG [StoreCloser-TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/C/cebc6cb5ab6449fdb5a5db4cd1fe73b7 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/C/cebc6cb5ab6449fdb5a5db4cd1fe73b7 2024-11-23T16:14:09,204 DEBUG [StoreCloser-TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/C/98ecc8312acd4a6280968eefea7966ea to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/C/98ecc8312acd4a6280968eefea7966ea 2024-11-23T16:14:09,204 DEBUG [StoreCloser-TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/C/7abd61d73ed44ac798a81202518d8dd8 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/C/7abd61d73ed44ac798a81202518d8dd8 2024-11-23T16:14:09,205 DEBUG [StoreCloser-TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/C/161c7cc7bd7849a186a599868d6362eb to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/C/161c7cc7bd7849a186a599868d6362eb 2024-11-23T16:14:09,206 DEBUG [StoreCloser-TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/C/b7af0fa5777b415486f6c1b77a4bd80f to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/C/b7af0fa5777b415486f6c1b77a4bd80f 2024-11-23T16:14:09,207 DEBUG [StoreCloser-TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/C/33f6b41fe17c4db0859f0a55a753a1d3 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/C/33f6b41fe17c4db0859f0a55a753a1d3 2024-11-23T16:14:09,212 DEBUG [RS_CLOSE_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/recovered.edits/552.seqid, newMaxSeqId=552, maxSeqId=1 2024-11-23T16:14:09,213 INFO [RS_CLOSE_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302. 2024-11-23T16:14:09,213 DEBUG [RS_CLOSE_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1676): Region close journal for 4550087eb34ce466d194f9ce96539302: Waiting for close lock at 1732378447817Running coprocessor pre-close hooks at 1732378447817Disabling compacts and flushes for region at 1732378447817Disabling writes for close at 1732378447818 (+1 ms)Obtaining lock to block concurrent updates at 1732378447818Preparing flush snapshotting stores in 4550087eb34ce466d194f9ce96539302 at 1732378447818Finished memstore snapshotting TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302., syncing WAL and waiting on mvcc, flushsize=dataSize=20610, getHeapSize=61920, getOffHeapSize=0, getCellsCount=450 at 1732378447818Flushing stores of TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302. at 1732378447819 (+1 ms)Flushing 4550087eb34ce466d194f9ce96539302/A: creating writer at 1732378447819Flushing 4550087eb34ce466d194f9ce96539302/A: appending metadata at 1732378447822 (+3 ms)Flushing 4550087eb34ce466d194f9ce96539302/A: closing flushed file at 1732378447822Flushing 4550087eb34ce466d194f9ce96539302/B: creating writer at 1732378448233 (+411 ms)Flushing 4550087eb34ce466d194f9ce96539302/B: appending metadata at 1732378448236 (+3 ms)Flushing 4550087eb34ce466d194f9ce96539302/B: closing flushed file at 1732378448236Flushing 4550087eb34ce466d194f9ce96539302/C: creating writer at 1732378448645 (+409 ms)Flushing 4550087eb34ce466d194f9ce96539302/C: appending metadata at 1732378448648 (+3 ms)Flushing 4550087eb34ce466d194f9ce96539302/C: closing flushed file at 1732378448648Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1caad1e6: reopening flushed file at 1732378449057 (+409 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5788758d: reopening flushed file at 1732378449062 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@76e774fa: reopening flushed file at 1732378449067 (+5 ms)Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=0 B/0 for 4550087eb34ce466d194f9ce96539302 in 1255ms, sequenceid=549, compaction requested=true at 1732378449073 (+6 ms)Writing region close event to WAL at 1732378449209 (+136 ms)Running coprocessor post-close hooks at 1732378449213 (+4 ms)Closed at 1732378449213 2024-11-23T16:14:09,215 INFO [RS_CLOSE_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] handler.UnassignRegionHandler(157): Closed 4550087eb34ce466d194f9ce96539302 2024-11-23T16:14:09,216 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=4550087eb34ce466d194f9ce96539302, regionState=CLOSED 2024-11-23T16:14:09,218 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=30, ppid=29, state=RUNNABLE, hasLock=false; CloseRegionProcedure 4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 because future has completed 2024-11-23T16:14:09,220 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=30, resume processing ppid=29 2024-11-23T16:14:09,220 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=29, state=SUCCESS, hasLock=false; CloseRegionProcedure 4550087eb34ce466d194f9ce96539302, server=1a7c5470b472,44715,1732378420375 in 1.5590 sec 2024-11-23T16:14:09,222 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=29, resume processing ppid=28 2024-11-23T16:14:09,222 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, ppid=28, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=4550087eb34ce466d194f9ce96539302, UNASSIGN in 1.5650 sec 2024-11-23T16:14:09,224 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=28, resume processing ppid=27 2024-11-23T16:14:09,224 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, ppid=27, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.5720 sec 2024-11-23T16:14:09,225 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732378449225"}]},"ts":"1732378449225"} 2024-11-23T16:14:09,226 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-23T16:14:09,226 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-23T16:14:09,228 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.5880 sec 2024-11-23T16:14:09,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-23T16:14:09,789 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-23T16:14:09,792 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45799 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-23T16:14:09,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45799 {}] procedure2.ProcedureExecutor(1139): Stored pid=31, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-23T16:14:09,797 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=31, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-23T16:14:09,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-23T16:14:09,798 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=31, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-23T16:14:09,802 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302 2024-11-23T16:14:09,806 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/A, FileablePath, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/B, FileablePath, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/C, FileablePath, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/recovered.edits] 2024-11-23T16:14:09,808 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/A/14345d6e6d604c3e91a9c231b145d6dc to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/A/14345d6e6d604c3e91a9c231b145d6dc 2024-11-23T16:14:09,810 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/A/17a58f50a89e4c1b83f14910b610c13c to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/A/17a58f50a89e4c1b83f14910b610c13c 2024-11-23T16:14:09,811 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/A/881aed0576c94eadb532eb3d6fae4a85 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/A/881aed0576c94eadb532eb3d6fae4a85 2024-11-23T16:14:09,814 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/B/235f6ae49a8a47a6a350d9a4aacbcabc to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/B/235f6ae49a8a47a6a350d9a4aacbcabc 2024-11-23T16:14:09,815 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/B/d37ad426d879496e872fc19a0058feb0 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/B/d37ad426d879496e872fc19a0058feb0 2024-11-23T16:14:09,816 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/B/f6628fef4f684bc78e2252b54d5065b5 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/B/f6628fef4f684bc78e2252b54d5065b5 2024-11-23T16:14:09,818 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/C/0876305e3b094ae5aedaf6d6927354d9 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/C/0876305e3b094ae5aedaf6d6927354d9 2024-11-23T16:14:09,820 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/C/4fcda3d692f1404c819e60acfb0064cd to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/C/4fcda3d692f1404c819e60acfb0064cd 2024-11-23T16:14:09,821 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/C/d729f51fbd5545b8882012637ad66fb3 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/C/d729f51fbd5545b8882012637ad66fb3 2024-11-23T16:14:09,824 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/recovered.edits/552.seqid to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302/recovered.edits/552.seqid 2024-11-23T16:14:09,824 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/4550087eb34ce466d194f9ce96539302 2024-11-23T16:14:09,824 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-23T16:14:09,829 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=31, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-23T16:14:09,833 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44715 {}] util.ReflectedFunctionCache(97): Populated cache for org.apache.hadoop.hbase.filter.KeyOnlyFilter in 0ms 2024-11-23T16:14:09,840 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-23T16:14:09,877 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-23T16:14:09,879 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=31, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-23T16:14:09,879 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-23T16:14:09,879 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732378449879"}]},"ts":"9223372036854775807"} 2024-11-23T16:14:09,883 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-23T16:14:09,883 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 4550087eb34ce466d194f9ce96539302, NAME => 'TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302.', STARTKEY => '', ENDKEY => ''}] 2024-11-23T16:14:09,883 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-23T16:14:09,884 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732378449883"}]},"ts":"9223372036854775807"} 2024-11-23T16:14:09,886 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-23T16:14:09,887 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=31, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-23T16:14:09,888 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 95 msec 2024-11-23T16:14:09,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-23T16:14:09,909 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-23T16:14:09,909 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-23T16:14:09,921 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMixedAtomicity Thread=248 (was 211) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-10 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/1a7c5470b472:0-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS:0;1a7c5470b472:44715-shortCompactions-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.PriorityBlockingQueue.take(PriorityBlockingQueue.java:535) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/1a7c5470b472:0-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/1a7c5470b472:0-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_CLOSE_REGION-regionserver/1a7c5470b472:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1268241868_22 at /127.0.0.1:34496 [Waiting for operation #330] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/1a7c5470b472:0-8 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-13 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_REGION-regionserver/1a7c5470b472:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-14 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-2057066486_22 at /127.0.0.1:50034 [Waiting for operation #339] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/1a7c5470b472:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-12 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-16 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/1a7c5470b472:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-15 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-11 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/1a7c5470b472:0-9 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/1a7c5470b472:0-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/1a7c5470b472:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/1a7c5470b472:0-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=453 (was 441) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=354 (was 198) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=7987 (was 7806) - AvailableMemoryMB LEAK? - 2024-11-23T16:14:09,930 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobMixedAtomicity Thread=248, OpenFileDescriptor=453, MaxFileDescriptor=1048576, SystemLoadAverage=354, ProcessCount=11, AvailableMemoryMB=7987 2024-11-23T16:14:09,932 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45799 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-23T16:14:09,932 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45799 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-23T16:14:09,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45799 {}] procedure2.ProcedureExecutor(1139): Stored pid=32, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-23T16:14:09,934 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-23T16:14:09,935 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:09,935 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45799 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 32 2024-11-23T16:14:09,935 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-23T16:14:09,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-23T16:14:09,941 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073741945_1121 (size=1061) 2024-11-23T16:14:10,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-23T16:14:10,108 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-23T16:14:10,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-23T16:14:10,344 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => cf5d4648fde06cb520ace381e0be9853, NAME => 'TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca 2024-11-23T16:14:10,349 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073741946_1122 (size=53) 2024-11-23T16:14:10,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-23T16:14:10,750 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T16:14:10,750 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing cf5d4648fde06cb520ace381e0be9853, disabling compactions & flushes 2024-11-23T16:14:10,750 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853. 2024-11-23T16:14:10,750 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853. 2024-11-23T16:14:10,750 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853. after waiting 0 ms 2024-11-23T16:14:10,750 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853. 2024-11-23T16:14:10,750 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853. 2024-11-23T16:14:10,750 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for cf5d4648fde06cb520ace381e0be9853: Waiting for close lock at 1732378450750Disabling compacts and flushes for region at 1732378450750Disabling writes for close at 1732378450750Writing region close event to WAL at 1732378450750Closed at 1732378450750 2024-11-23T16:14:10,751 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-23T16:14:10,752 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732378450752"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732378450752"}]},"ts":"1732378450752"} 2024-11-23T16:14:10,754 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-23T16:14:10,754 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-23T16:14:10,755 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732378450754"}]},"ts":"1732378450754"} 2024-11-23T16:14:10,756 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-23T16:14:10,756 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=cf5d4648fde06cb520ace381e0be9853, ASSIGN}] 2024-11-23T16:14:10,758 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=cf5d4648fde06cb520ace381e0be9853, ASSIGN 2024-11-23T16:14:10,758 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=cf5d4648fde06cb520ace381e0be9853, ASSIGN; state=OFFLINE, location=1a7c5470b472,44715,1732378420375; forceNewPlan=false, retain=false 2024-11-23T16:14:10,909 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=33 updating hbase:meta row=cf5d4648fde06cb520ace381e0be9853, regionState=OPENING, regionLocation=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:10,911 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=cf5d4648fde06cb520ace381e0be9853, ASSIGN because future has completed 2024-11-23T16:14:10,912 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=34, ppid=33, state=RUNNABLE, hasLock=false; OpenRegionProcedure cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375}] 2024-11-23T16:14:11,068 INFO [RS_OPEN_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853. 2024-11-23T16:14:11,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-23T16:14:11,068 DEBUG [RS_OPEN_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7752): Opening region: {ENCODED => cf5d4648fde06cb520ace381e0be9853, NAME => 'TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853.', STARTKEY => '', ENDKEY => ''} 2024-11-23T16:14:11,068 DEBUG [RS_OPEN_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees cf5d4648fde06cb520ace381e0be9853 2024-11-23T16:14:11,068 DEBUG [RS_OPEN_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T16:14:11,068 DEBUG [RS_OPEN_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7794): checking encryption for cf5d4648fde06cb520ace381e0be9853 2024-11-23T16:14:11,068 DEBUG [RS_OPEN_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7797): checking classloading for cf5d4648fde06cb520ace381e0be9853 2024-11-23T16:14:11,070 INFO [StoreOpener-cf5d4648fde06cb520ace381e0be9853-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region cf5d4648fde06cb520ace381e0be9853 2024-11-23T16:14:11,071 INFO [StoreOpener-cf5d4648fde06cb520ace381e0be9853-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-23T16:14:11,072 INFO [StoreOpener-cf5d4648fde06cb520ace381e0be9853-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region cf5d4648fde06cb520ace381e0be9853 columnFamilyName A 2024-11-23T16:14:11,072 DEBUG [StoreOpener-cf5d4648fde06cb520ace381e0be9853-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:11,072 INFO [StoreOpener-cf5d4648fde06cb520ace381e0be9853-1 {}] regionserver.HStore(327): Store=cf5d4648fde06cb520ace381e0be9853/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T16:14:11,073 INFO [StoreOpener-cf5d4648fde06cb520ace381e0be9853-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region cf5d4648fde06cb520ace381e0be9853 2024-11-23T16:14:11,073 INFO [StoreOpener-cf5d4648fde06cb520ace381e0be9853-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-23T16:14:11,074 INFO [StoreOpener-cf5d4648fde06cb520ace381e0be9853-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region cf5d4648fde06cb520ace381e0be9853 columnFamilyName B 2024-11-23T16:14:11,074 DEBUG [StoreOpener-cf5d4648fde06cb520ace381e0be9853-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:11,074 INFO [StoreOpener-cf5d4648fde06cb520ace381e0be9853-1 {}] regionserver.HStore(327): Store=cf5d4648fde06cb520ace381e0be9853/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T16:14:11,074 INFO [StoreOpener-cf5d4648fde06cb520ace381e0be9853-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region cf5d4648fde06cb520ace381e0be9853 2024-11-23T16:14:11,075 INFO [StoreOpener-cf5d4648fde06cb520ace381e0be9853-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-23T16:14:11,075 INFO [StoreOpener-cf5d4648fde06cb520ace381e0be9853-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region cf5d4648fde06cb520ace381e0be9853 columnFamilyName C 2024-11-23T16:14:11,075 DEBUG [StoreOpener-cf5d4648fde06cb520ace381e0be9853-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:11,076 INFO [StoreOpener-cf5d4648fde06cb520ace381e0be9853-1 {}] regionserver.HStore(327): Store=cf5d4648fde06cb520ace381e0be9853/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T16:14:11,076 INFO [RS_OPEN_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853. 2024-11-23T16:14:11,076 DEBUG [RS_OPEN_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1038): replaying wal for cf5d4648fde06cb520ace381e0be9853 2024-11-23T16:14:11,077 DEBUG [RS_OPEN_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853 2024-11-23T16:14:11,077 DEBUG [RS_OPEN_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853 2024-11-23T16:14:11,078 DEBUG [RS_OPEN_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1048): stopping wal replay for cf5d4648fde06cb520ace381e0be9853 2024-11-23T16:14:11,078 DEBUG [RS_OPEN_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1060): Cleaning up temporary data for cf5d4648fde06cb520ace381e0be9853 2024-11-23T16:14:11,078 DEBUG [RS_OPEN_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-23T16:14:11,079 DEBUG [RS_OPEN_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1093): writing seq id for cf5d4648fde06cb520ace381e0be9853 2024-11-23T16:14:11,081 DEBUG [RS_OPEN_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-23T16:14:11,081 INFO [RS_OPEN_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1114): Opened cf5d4648fde06cb520ace381e0be9853; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69610843, jitterRate=0.037282392382621765}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-23T16:14:11,082 DEBUG [RS_OPEN_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1122): Running coprocessor post-open hooks for cf5d4648fde06cb520ace381e0be9853 2024-11-23T16:14:11,082 DEBUG [RS_OPEN_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1006): Region open journal for cf5d4648fde06cb520ace381e0be9853: Running coprocessor pre-open hook at 1732378451069Writing region info on filesystem at 1732378451069Initializing all the Stores at 1732378451069Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732378451069Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732378451070 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732378451070Cleaning up temporary data from old regions at 1732378451078 (+8 ms)Running coprocessor post-open hooks at 1732378451082 (+4 ms)Region opened successfully at 1732378451082 2024-11-23T16:14:11,083 INFO [RS_OPEN_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., pid=34, masterSystemTime=1732378451064 2024-11-23T16:14:11,085 DEBUG [RS_OPEN_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853. 2024-11-23T16:14:11,085 INFO [RS_OPEN_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853. 2024-11-23T16:14:11,086 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=33 updating hbase:meta row=cf5d4648fde06cb520ace381e0be9853, regionState=OPEN, openSeqNum=2, regionLocation=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:11,087 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=34, ppid=33, state=RUNNABLE, hasLock=false; OpenRegionProcedure cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 because future has completed 2024-11-23T16:14:11,090 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=34, resume processing ppid=33 2024-11-23T16:14:11,090 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, ppid=33, state=SUCCESS, hasLock=false; OpenRegionProcedure cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 in 176 msec 2024-11-23T16:14:11,092 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=33, resume processing ppid=32 2024-11-23T16:14:11,092 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=cf5d4648fde06cb520ace381e0be9853, ASSIGN in 334 msec 2024-11-23T16:14:11,093 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-23T16:14:11,093 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732378451093"}]},"ts":"1732378451093"} 2024-11-23T16:14:11,094 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-23T16:14:11,095 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-23T16:14:11,096 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1630 sec 2024-11-23T16:14:12,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-23T16:14:12,079 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-23T16:14:12,079 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@79527845, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T16:14:12,080 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1a7c5470b472,45799,-1 for getting cluster id 2024-11-23T16:14:12,080 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-23T16:14:12,081 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '34e0592b-548a-4d73-be6a-df1bafadfd92' 2024-11-23T16:14:12,081 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-23T16:14:12,082 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "34e0592b-548a-4d73-be6a-df1bafadfd92" 2024-11-23T16:14:12,082 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7287a320, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T16:14:12,082 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1a7c5470b472,45799,-1] 2024-11-23T16:14:12,082 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-23T16:14:12,082 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T16:14:12,084 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55266, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-23T16:14:12,084 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@641dcc92, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T16:14:12,085 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T16:14:12,086 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1a7c5470b472,44715,1732378420375, seqNum=-1] 2024-11-23T16:14:12,086 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T16:14:12,087 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59572, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T16:14:12,090 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-23T16:14:12,090 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncConnectionImpl(321): The fetched master address is 1a7c5470b472,45799,1732378419617 2024-11-23T16:14:12,091 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@296fe578 2024-11-23T16:14:12,091 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-23T16:14:12,092 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55280, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-23T16:14:12,096 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45799 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-23T16:14:12,097 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45799 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-23T16:14:12,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45799 {}] procedure2.ProcedureExecutor(1139): Stored pid=35, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-23T16:14:12,114 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073741947_1123 (size=1097) 2024-11-23T16:14:12,516 DEBUG [PEWorker-1 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-11-23T16:14:12,516 INFO [PEWorker-1 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-11-23T16:14:12,520 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=36, ppid=35, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-23T16:14:12,526 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=cf5d4648fde06cb520ace381e0be9853, REOPEN/MOVE}] 2024-11-23T16:14:12,526 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=cf5d4648fde06cb520ace381e0be9853, REOPEN/MOVE 2024-11-23T16:14:12,527 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=37 updating hbase:meta row=cf5d4648fde06cb520ace381e0be9853, regionState=CLOSING, regionLocation=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:12,529 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=cf5d4648fde06cb520ace381e0be9853, REOPEN/MOVE because future has completed 2024-11-23T16:14:12,529 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-23T16:14:12,529 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=38, ppid=37, state=RUNNABLE, hasLock=false; CloseRegionProcedure cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375}] 2024-11-23T16:14:12,682 INFO [RS_CLOSE_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] handler.UnassignRegionHandler(122): Close cf5d4648fde06cb520ace381e0be9853 2024-11-23T16:14:12,682 DEBUG [RS_CLOSE_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-23T16:14:12,682 DEBUG [RS_CLOSE_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1722): Closing cf5d4648fde06cb520ace381e0be9853, disabling compactions & flushes 2024-11-23T16:14:12,682 INFO [RS_CLOSE_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853. 2024-11-23T16:14:12,682 DEBUG [RS_CLOSE_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853. 2024-11-23T16:14:12,682 DEBUG [RS_CLOSE_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853. after waiting 0 ms 2024-11-23T16:14:12,682 DEBUG [RS_CLOSE_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853. 2024-11-23T16:14:12,686 DEBUG [RS_CLOSE_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-23T16:14:12,687 INFO [RS_CLOSE_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853. 2024-11-23T16:14:12,687 DEBUG [RS_CLOSE_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1676): Region close journal for cf5d4648fde06cb520ace381e0be9853: Waiting for close lock at 1732378452682Running coprocessor pre-close hooks at 1732378452682Disabling compacts and flushes for region at 1732378452682Disabling writes for close at 1732378452682Writing region close event to WAL at 1732378452683 (+1 ms)Running coprocessor post-close hooks at 1732378452687 (+4 ms)Closed at 1732378452687 2024-11-23T16:14:12,687 WARN [RS_CLOSE_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegionServer(3299): Not adding moved region record: cf5d4648fde06cb520ace381e0be9853 to self. 2024-11-23T16:14:12,689 INFO [RS_CLOSE_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] handler.UnassignRegionHandler(157): Closed cf5d4648fde06cb520ace381e0be9853 2024-11-23T16:14:12,689 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=37 updating hbase:meta row=cf5d4648fde06cb520ace381e0be9853, regionState=CLOSED 2024-11-23T16:14:12,691 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=38, ppid=37, state=RUNNABLE, hasLock=false; CloseRegionProcedure cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 because future has completed 2024-11-23T16:14:12,694 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=38, resume processing ppid=37 2024-11-23T16:14:12,694 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=38, ppid=37, state=SUCCESS, hasLock=false; CloseRegionProcedure cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 in 163 msec 2024-11-23T16:14:12,695 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=cf5d4648fde06cb520ace381e0be9853, REOPEN/MOVE; state=CLOSED, location=1a7c5470b472,44715,1732378420375; forceNewPlan=false, retain=true 2024-11-23T16:14:12,845 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=37 updating hbase:meta row=cf5d4648fde06cb520ace381e0be9853, regionState=OPENING, regionLocation=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:12,847 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=cf5d4648fde06cb520ace381e0be9853, REOPEN/MOVE because future has completed 2024-11-23T16:14:12,847 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=39, ppid=37, state=RUNNABLE, hasLock=false; OpenRegionProcedure cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375}] 2024-11-23T16:14:13,003 INFO [RS_OPEN_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853. 2024-11-23T16:14:13,003 DEBUG [RS_OPEN_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7752): Opening region: {ENCODED => cf5d4648fde06cb520ace381e0be9853, NAME => 'TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853.', STARTKEY => '', ENDKEY => ''} 2024-11-23T16:14:13,003 DEBUG [RS_OPEN_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees cf5d4648fde06cb520ace381e0be9853 2024-11-23T16:14:13,003 DEBUG [RS_OPEN_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T16:14:13,003 DEBUG [RS_OPEN_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7794): checking encryption for cf5d4648fde06cb520ace381e0be9853 2024-11-23T16:14:13,003 DEBUG [RS_OPEN_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7797): checking classloading for cf5d4648fde06cb520ace381e0be9853 2024-11-23T16:14:13,005 INFO [StoreOpener-cf5d4648fde06cb520ace381e0be9853-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region cf5d4648fde06cb520ace381e0be9853 2024-11-23T16:14:13,006 INFO [StoreOpener-cf5d4648fde06cb520ace381e0be9853-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-23T16:14:13,011 INFO [StoreOpener-cf5d4648fde06cb520ace381e0be9853-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region cf5d4648fde06cb520ace381e0be9853 columnFamilyName A 2024-11-23T16:14:13,012 DEBUG [StoreOpener-cf5d4648fde06cb520ace381e0be9853-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:13,013 INFO [StoreOpener-cf5d4648fde06cb520ace381e0be9853-1 {}] regionserver.HStore(327): Store=cf5d4648fde06cb520ace381e0be9853/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T16:14:13,013 INFO [StoreOpener-cf5d4648fde06cb520ace381e0be9853-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region cf5d4648fde06cb520ace381e0be9853 2024-11-23T16:14:13,014 INFO [StoreOpener-cf5d4648fde06cb520ace381e0be9853-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-23T16:14:13,014 INFO [StoreOpener-cf5d4648fde06cb520ace381e0be9853-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region cf5d4648fde06cb520ace381e0be9853 columnFamilyName B 2024-11-23T16:14:13,014 DEBUG [StoreOpener-cf5d4648fde06cb520ace381e0be9853-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:13,014 INFO [StoreOpener-cf5d4648fde06cb520ace381e0be9853-1 {}] regionserver.HStore(327): Store=cf5d4648fde06cb520ace381e0be9853/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T16:14:13,014 INFO [StoreOpener-cf5d4648fde06cb520ace381e0be9853-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region cf5d4648fde06cb520ace381e0be9853 2024-11-23T16:14:13,015 INFO [StoreOpener-cf5d4648fde06cb520ace381e0be9853-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-23T16:14:13,015 INFO [StoreOpener-cf5d4648fde06cb520ace381e0be9853-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region cf5d4648fde06cb520ace381e0be9853 columnFamilyName C 2024-11-23T16:14:13,015 DEBUG [StoreOpener-cf5d4648fde06cb520ace381e0be9853-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:13,016 INFO [StoreOpener-cf5d4648fde06cb520ace381e0be9853-1 {}] regionserver.HStore(327): Store=cf5d4648fde06cb520ace381e0be9853/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T16:14:13,016 INFO [RS_OPEN_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853. 2024-11-23T16:14:13,016 DEBUG [RS_OPEN_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1038): replaying wal for cf5d4648fde06cb520ace381e0be9853 2024-11-23T16:14:13,016 DEBUG [RS_OPEN_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853 2024-11-23T16:14:13,017 DEBUG [RS_OPEN_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853 2024-11-23T16:14:13,018 DEBUG [RS_OPEN_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1048): stopping wal replay for cf5d4648fde06cb520ace381e0be9853 2024-11-23T16:14:13,018 DEBUG [RS_OPEN_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1060): Cleaning up temporary data for cf5d4648fde06cb520ace381e0be9853 2024-11-23T16:14:13,018 DEBUG [RS_OPEN_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-23T16:14:13,020 DEBUG [RS_OPEN_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1093): writing seq id for cf5d4648fde06cb520ace381e0be9853 2024-11-23T16:14:13,020 INFO [RS_OPEN_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1114): Opened cf5d4648fde06cb520ace381e0be9853; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71945788, jitterRate=0.07207578420639038}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-23T16:14:13,021 DEBUG [RS_OPEN_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1122): Running coprocessor post-open hooks for cf5d4648fde06cb520ace381e0be9853 2024-11-23T16:14:13,021 DEBUG [RS_OPEN_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1006): Region open journal for cf5d4648fde06cb520ace381e0be9853: Running coprocessor pre-open hook at 1732378453003Writing region info on filesystem at 1732378453003Initializing all the Stores at 1732378453004 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732378453004Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732378453004Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732378453004Cleaning up temporary data from old regions at 1732378453018 (+14 ms)Running coprocessor post-open hooks at 1732378453021 (+3 ms)Region opened successfully at 1732378453021 2024-11-23T16:14:13,022 INFO [RS_OPEN_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., pid=39, masterSystemTime=1732378452999 2024-11-23T16:14:13,023 DEBUG [RS_OPEN_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853. 2024-11-23T16:14:13,023 INFO [RS_OPEN_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853. 2024-11-23T16:14:13,024 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=37 updating hbase:meta row=cf5d4648fde06cb520ace381e0be9853, regionState=OPEN, openSeqNum=5, regionLocation=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:13,025 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=39, ppid=37, state=RUNNABLE, hasLock=false; OpenRegionProcedure cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 because future has completed 2024-11-23T16:14:13,028 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=39, resume processing ppid=37 2024-11-23T16:14:13,028 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, ppid=37, state=SUCCESS, hasLock=false; OpenRegionProcedure cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 in 179 msec 2024-11-23T16:14:13,029 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=37, resume processing ppid=36 2024-11-23T16:14:13,030 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, ppid=36, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=cf5d4648fde06cb520ace381e0be9853, REOPEN/MOVE in 502 msec 2024-11-23T16:14:13,032 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=36, resume processing ppid=35 2024-11-23T16:14:13,032 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, ppid=35, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 511 msec 2024-11-23T16:14:13,035 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 934 msec 2024-11-23T16:14:13,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=35 2024-11-23T16:14:13,037 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-23T16:14:13,038 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5adc4e02, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T16:14:13,038 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1a7c5470b472,45799,-1 for getting cluster id 2024-11-23T16:14:13,038 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-23T16:14:13,039 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '34e0592b-548a-4d73-be6a-df1bafadfd92' 2024-11-23T16:14:13,039 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-23T16:14:13,039 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "34e0592b-548a-4d73-be6a-df1bafadfd92" 2024-11-23T16:14:13,039 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5606186b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T16:14:13,039 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1a7c5470b472,45799,-1] 2024-11-23T16:14:13,039 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-23T16:14:13,039 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T16:14:13,040 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55294, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-23T16:14:13,041 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@43b9f6e3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T16:14:13,041 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@681c9b14, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T16:14:13,041 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1a7c5470b472,45799,-1 for getting cluster id 2024-11-23T16:14:13,041 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-23T16:14:13,042 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '34e0592b-548a-4d73-be6a-df1bafadfd92' 2024-11-23T16:14:13,042 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-23T16:14:13,042 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "34e0592b-548a-4d73-be6a-df1bafadfd92" 2024-11-23T16:14:13,043 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2ad43a4e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T16:14:13,043 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1a7c5470b472,45799,-1] 2024-11-23T16:14:13,043 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-23T16:14:13,043 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T16:14:13,044 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55314, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-23T16:14:13,044 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2d5793df, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T16:14:13,045 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@51d91e8d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T16:14:13,045 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1a7c5470b472,45799,-1 for getting cluster id 2024-11-23T16:14:13,045 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-23T16:14:13,046 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '34e0592b-548a-4d73-be6a-df1bafadfd92' 2024-11-23T16:14:13,046 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-23T16:14:13,046 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "34e0592b-548a-4d73-be6a-df1bafadfd92" 2024-11-23T16:14:13,046 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@349991e9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T16:14:13,046 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1a7c5470b472,45799,-1] 2024-11-23T16:14:13,046 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-23T16:14:13,046 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T16:14:13,047 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55330, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-23T16:14:13,047 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@12e0acd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T16:14:13,048 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@771f5e02, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T16:14:13,048 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1a7c5470b472,45799,-1 for getting cluster id 2024-11-23T16:14:13,048 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-23T16:14:13,049 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '34e0592b-548a-4d73-be6a-df1bafadfd92' 2024-11-23T16:14:13,049 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-23T16:14:13,049 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "34e0592b-548a-4d73-be6a-df1bafadfd92" 2024-11-23T16:14:13,049 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a6bb083, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T16:14:13,050 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1a7c5470b472,45799,-1] 2024-11-23T16:14:13,050 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-23T16:14:13,050 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T16:14:13,051 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55360, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-23T16:14:13,051 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3ffffc00, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T16:14:13,052 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@37b75e2b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T16:14:13,052 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1a7c5470b472,45799,-1 for getting cluster id 2024-11-23T16:14:13,052 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-23T16:14:13,053 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '34e0592b-548a-4d73-be6a-df1bafadfd92' 2024-11-23T16:14:13,053 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-23T16:14:13,053 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "34e0592b-548a-4d73-be6a-df1bafadfd92" 2024-11-23T16:14:13,053 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3ca17a44, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T16:14:13,053 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1a7c5470b472,45799,-1] 2024-11-23T16:14:13,053 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-23T16:14:13,053 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T16:14:13,054 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55388, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-23T16:14:13,055 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@750dfbed, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T16:14:13,055 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f5453a4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T16:14:13,055 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1a7c5470b472,45799,-1 for getting cluster id 2024-11-23T16:14:13,056 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-23T16:14:13,056 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '34e0592b-548a-4d73-be6a-df1bafadfd92' 2024-11-23T16:14:13,056 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-23T16:14:13,056 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "34e0592b-548a-4d73-be6a-df1bafadfd92" 2024-11-23T16:14:13,057 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@488206fc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T16:14:13,057 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1a7c5470b472,45799,-1] 2024-11-23T16:14:13,057 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-23T16:14:13,057 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T16:14:13,057 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55416, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-23T16:14:13,058 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7024a902, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T16:14:13,059 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@73b6e690, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T16:14:13,059 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1a7c5470b472,45799,-1 for getting cluster id 2024-11-23T16:14:13,059 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-23T16:14:13,059 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '34e0592b-548a-4d73-be6a-df1bafadfd92' 2024-11-23T16:14:13,060 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-23T16:14:13,060 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "34e0592b-548a-4d73-be6a-df1bafadfd92" 2024-11-23T16:14:13,060 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2e8d81e4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T16:14:13,060 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1a7c5470b472,45799,-1] 2024-11-23T16:14:13,060 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-23T16:14:13,060 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T16:14:13,061 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55422, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-23T16:14:13,061 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7b0905f9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T16:14:13,062 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5bd05ea8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T16:14:13,062 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1a7c5470b472,45799,-1 for getting cluster id 2024-11-23T16:14:13,062 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-23T16:14:13,063 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '34e0592b-548a-4d73-be6a-df1bafadfd92' 2024-11-23T16:14:13,063 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-23T16:14:13,063 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "34e0592b-548a-4d73-be6a-df1bafadfd92" 2024-11-23T16:14:13,063 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@21c3e521, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T16:14:13,063 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1a7c5470b472,45799,-1] 2024-11-23T16:14:13,063 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-23T16:14:13,063 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T16:14:13,064 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55438, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-23T16:14:13,064 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@34f1458f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T16:14:13,065 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1dff183a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T16:14:13,065 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1a7c5470b472,45799,-1 for getting cluster id 2024-11-23T16:14:13,065 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-23T16:14:13,065 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '34e0592b-548a-4d73-be6a-df1bafadfd92' 2024-11-23T16:14:13,066 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-23T16:14:13,066 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "34e0592b-548a-4d73-be6a-df1bafadfd92" 2024-11-23T16:14:13,066 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6489337f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T16:14:13,066 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1a7c5470b472,45799,-1] 2024-11-23T16:14:13,066 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-23T16:14:13,066 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T16:14:13,067 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55466, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-23T16:14:13,067 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@442cd356, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T16:14:13,068 DEBUG [Thread-628 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T16:14:13,068 DEBUG [Thread-639 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T16:14:13,068 DEBUG [Thread-630 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T16:14:13,068 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45799 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-23T16:14:13,069 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1a7c5470b472,44715,1732378420375, seqNum=-1] 2024-11-23T16:14:13,069 DEBUG [Thread-635 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T16:14:13,069 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1a7c5470b472,44715,1732378420375, seqNum=-1] 2024-11-23T16:14:13,069 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1a7c5470b472,44715,1732378420375, seqNum=-1] 2024-11-23T16:14:13,069 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T16:14:13,069 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T16:14:13,069 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T16:14:13,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45799 {}] procedure2.ProcedureExecutor(1139): Stored pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees 2024-11-23T16:14:13,070 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1a7c5470b472,44715,1732378420375, seqNum=-1] 2024-11-23T16:14:13,070 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59584, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T16:14:13,071 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T16:14:13,071 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59586, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T16:14:13,071 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-23T16:14:13,071 DEBUG [Thread-622 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T16:14:13,071 DEBUG [Thread-633 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T16:14:13,072 DEBUG [Thread-626 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T16:14:13,072 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59596, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T16:14:13,072 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59594, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T16:14:13,072 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1a7c5470b472,44715,1732378420375, seqNum=-1] 2024-11-23T16:14:13,072 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-23T16:14:13,072 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1a7c5470b472,44715,1732378420375, seqNum=-1] 2024-11-23T16:14:13,072 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=41, ppid=40, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-23T16:14:13,072 DEBUG [Thread-637 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T16:14:13,073 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T16:14:13,073 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T16:14:13,073 DEBUG [Thread-624 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T16:14:13,074 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59598, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T16:14:13,074 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1a7c5470b472,44715,1732378420375, seqNum=-1] 2024-11-23T16:14:13,074 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59614, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T16:14:13,074 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1a7c5470b472,44715,1732378420375, seqNum=-1] 2024-11-23T16:14:13,074 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T16:14:13,074 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T16:14:13,075 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1a7c5470b472,44715,1732378420375, seqNum=-1] 2024-11-23T16:14:13,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-23T16:14:13,075 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T16:14:13,076 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59620, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T16:14:13,076 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5] 2024-11-23T16:14:13,076 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5] 2024-11-23T16:14:13,076 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59628, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T16:14:13,077 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5] 2024-11-23T16:14:13,077 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59638, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T16:14:13,078 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5] 2024-11-23T16:14:13,079 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5] 2024-11-23T16:14:13,080 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5] 2024-11-23T16:14:13,081 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5] 2024-11-23T16:14:13,082 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5] 2024-11-23T16:14:13,084 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5] 2024-11-23T16:14:13,094 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cf5d4648fde06cb520ace381e0be9853 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-23T16:14:13,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(8855): Flush requested on cf5d4648fde06cb520ace381e0be9853 2024-11-23T16:14:13,096 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf5d4648fde06cb520ace381e0be9853, store=A 2024-11-23T16:14:13,097 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-23T16:14:13,097 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf5d4648fde06cb520ace381e0be9853, store=B 2024-11-23T16:14:13,097 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-23T16:14:13,097 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf5d4648fde06cb520ace381e0be9853, store=C 2024-11-23T16:14:13,097 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-23T16:14:13,148 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411235f6b7ad87d124dcdb65f108724ce0548_cf5d4648fde06cb520ace381e0be9853 is 50, key is test_row_0/A:col10/1732378453089/Put/seqid=0 2024-11-23T16:14:13,160 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073741948_1124 (size=19474) 2024-11-23T16:14:13,165 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:13,168 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1a7c5470b472:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732378453129, max timestamp=1732378453163Num uniques 859; 2024-11-23T16:14:13,168 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1a7c5470b472:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732378453129, max timestamp=1732378453163Num uniques 859; 2024-11-23T16:14:13,168 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1a7c5470b472:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-23T16:14:13,168 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1a7c5470b472:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-23T16:14:13,172 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1a7c5470b472:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732378453129, max timestamp=1732378453163Num uniques 859; 2024-11-23T16:14:13,172 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1a7c5470b472:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-23T16:14:13,172 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411235f6b7ad87d124dcdb65f108724ce0548_cf5d4648fde06cb520ace381e0be9853 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411235f6b7ad87d124dcdb65f108724ce0548_cf5d4648fde06cb520ace381e0be9853 2024-11-23T16:14:13,174 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/.tmp/A/389baa2344bd4e88a84af99676683d7c, store: [table=TestAcidGuarantees family=A region=cf5d4648fde06cb520ace381e0be9853] 2024-11-23T16:14:13,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-23T16:14:13,185 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/.tmp/A/389baa2344bd4e88a84af99676683d7c is 175, key is test_row_0/A:col10/1732378453089/Put/seqid=0 2024-11-23T16:14:13,199 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073741949_1125 (size=56733) 2024-11-23T16:14:13,203 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=15, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/.tmp/A/389baa2344bd4e88a84af99676683d7c 2024-11-23T16:14:13,230 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-23T16:14:13,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853. 2024-11-23T16:14:13,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853. as already flushing 2024-11-23T16:14:13,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853. 2024-11-23T16:14:13,230 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => cf5d4648fde06cb520ace381e0be9853, NAME => 'TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:14:13,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => cf5d4648fde06cb520ace381e0be9853, NAME => 'TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:14:13,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cf5d4648fde06cb520ace381e0be9853, NAME => 'TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cf5d4648fde06cb520ace381e0be9853, NAME => 'TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:14:13,237 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/.tmp/B/a4ba58cfe0284efa8306d9f0f422a16c is 50, key is test_row_0/B:col10/1732378453089/Put/seqid=0 2024-11-23T16:14:13,246 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073741950_1126 (size=12001) 2024-11-23T16:14:13,246 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:13,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59614 deadline: 1732378513234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:13,246 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/.tmp/B/a4ba58cfe0284efa8306d9f0f422a16c 2024-11-23T16:14:13,247 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:13,248 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:13,248 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:14:13,250 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:13,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59584 deadline: 1732378513240, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:13,251 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:13,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59638 deadline: 1732378513242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:13,252 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:13,252 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:13,252 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:14:13,252 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:13,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59628 deadline: 1732378513244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:13,253 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:13,253 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:13,253 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:13,253 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:13,253 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:14:13,253 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:14:13,253 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:13,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59586 deadline: 1732378513248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:13,254 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:13,254 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:13,254 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:14:13,283 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/.tmp/C/5bf42799999d403d8173ae3cd6a75c07 is 50, key is test_row_0/C:col10/1732378453089/Put/seqid=0 2024-11-23T16:14:13,298 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073741951_1127 (size=12001) 2024-11-23T16:14:13,299 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/.tmp/C/5bf42799999d403d8173ae3cd6a75c07 2024-11-23T16:14:13,306 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/.tmp/A/389baa2344bd4e88a84af99676683d7c as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/A/389baa2344bd4e88a84af99676683d7c 2024-11-23T16:14:13,312 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/A/389baa2344bd4e88a84af99676683d7c, entries=300, sequenceid=15, filesize=55.4 K 2024-11-23T16:14:13,313 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/.tmp/B/a4ba58cfe0284efa8306d9f0f422a16c as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/B/a4ba58cfe0284efa8306d9f0f422a16c 2024-11-23T16:14:13,322 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/B/a4ba58cfe0284efa8306d9f0f422a16c, entries=150, sequenceid=15, filesize=11.7 K 2024-11-23T16:14:13,324 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/.tmp/C/5bf42799999d403d8173ae3cd6a75c07 as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/C/5bf42799999d403d8173ae3cd6a75c07 2024-11-23T16:14:13,333 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/C/5bf42799999d403d8173ae3cd6a75c07, entries=150, sequenceid=15, filesize=11.7 K 2024-11-23T16:14:13,334 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for cf5d4648fde06cb520ace381e0be9853 in 240ms, sequenceid=15, compaction requested=false 2024-11-23T16:14:13,335 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cf5d4648fde06cb520ace381e0be9853: 2024-11-23T16:14:13,368 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cf5d4648fde06cb520ace381e0be9853 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-11-23T16:14:13,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(8855): Flush requested on cf5d4648fde06cb520ace381e0be9853 2024-11-23T16:14:13,370 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf5d4648fde06cb520ace381e0be9853, store=A 2024-11-23T16:14:13,370 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-23T16:14:13,371 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf5d4648fde06cb520ace381e0be9853, store=B 2024-11-23T16:14:13,371 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-23T16:14:13,371 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf5d4648fde06cb520ace381e0be9853, store=C 2024-11-23T16:14:13,371 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-23T16:14:13,383 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-23T16:14:13,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853. 2024-11-23T16:14:13,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853. as already flushing 2024-11-23T16:14:13,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853. 2024-11-23T16:14:13,383 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => cf5d4648fde06cb520ace381e0be9853, NAME => 'TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:14:13,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => cf5d4648fde06cb520ace381e0be9853, NAME => 'TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:14:13,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cf5d4648fde06cb520ace381e0be9853, NAME => 'TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cf5d4648fde06cb520ace381e0be9853, NAME => 'TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:14:13,385 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241123a10252a8bf824235a8a7119cdcade2c3_cf5d4648fde06cb520ace381e0be9853 is 50, key is test_row_0/A:col10/1732378453368/Put/seqid=0 2024-11-23T16:14:13,387 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:13,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59584 deadline: 1732378513377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:13,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-23T16:14:13,388 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:13,389 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:13,389 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:14:13,389 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:13,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59586 deadline: 1732378513377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:13,390 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:13,390 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:13,390 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:14:13,396 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073741952_1128 (size=12154) 2024-11-23T16:14:13,396 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:13,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59638 deadline: 1732378513386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:13,397 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:13,397 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:13,397 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:14:13,398 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:13,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59614 deadline: 1732378513386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:13,398 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:13,398 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:13,398 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:14:13,399 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:13,399 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:13,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59628 deadline: 1732378513388, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:13,399 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:13,399 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:13,399 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:14:13,404 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241123a10252a8bf824235a8a7119cdcade2c3_cf5d4648fde06cb520ace381e0be9853 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123a10252a8bf824235a8a7119cdcade2c3_cf5d4648fde06cb520ace381e0be9853 2024-11-23T16:14:13,405 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/.tmp/A/a8c0de2db7aa45a8b430bae628f15ee0, store: [table=TestAcidGuarantees family=A region=cf5d4648fde06cb520ace381e0be9853] 2024-11-23T16:14:13,406 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/.tmp/A/a8c0de2db7aa45a8b430bae628f15ee0 is 175, key is test_row_0/A:col10/1732378453368/Put/seqid=0 2024-11-23T16:14:13,418 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073741953_1129 (size=30955) 2024-11-23T16:14:13,419 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=53, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/.tmp/A/a8c0de2db7aa45a8b430bae628f15ee0 2024-11-23T16:14:13,436 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/.tmp/B/0eaab5d5995441b5aa72f557fd1ea17d is 50, key is test_row_0/B:col10/1732378453368/Put/seqid=0 2024-11-23T16:14:13,446 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073741954_1130 (size=12001) 2024-11-23T16:14:13,449 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=53 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/.tmp/B/0eaab5d5995441b5aa72f557fd1ea17d 2024-11-23T16:14:13,464 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/.tmp/C/14c9ba67b3cd4082bdd8dab406b10fbb is 50, key is test_row_0/C:col10/1732378453368/Put/seqid=0 2024-11-23T16:14:13,472 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073741955_1131 (size=12001) 2024-11-23T16:14:13,501 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:13,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59584 deadline: 1732378513498, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:13,502 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:13,502 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:13,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59586 deadline: 1732378513498, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:13,502 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:13,502 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:14:13,502 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:13,503 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:13,503 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:14:13,512 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:13,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59638 deadline: 1732378513508, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:13,513 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:13,513 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:13,513 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:13,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59614 deadline: 1732378513509, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:13,513 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:14:13,513 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:13,513 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:13,513 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:13,513 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:14:13,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59628 deadline: 1732378513509, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:13,514 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:13,514 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:13,514 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:14:13,537 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-23T16:14:13,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853. 2024-11-23T16:14:13,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853. as already flushing 2024-11-23T16:14:13,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853. 2024-11-23T16:14:13,538 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => cf5d4648fde06cb520ace381e0be9853, NAME => 'TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:14:13,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => cf5d4648fde06cb520ace381e0be9853, NAME => 'TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:14:13,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cf5d4648fde06cb520ace381e0be9853, NAME => 'TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cf5d4648fde06cb520ace381e0be9853, NAME => 'TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:14:13,691 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-23T16:14:13,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853. 2024-11-23T16:14:13,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853. as already flushing 2024-11-23T16:14:13,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853. 2024-11-23T16:14:13,691 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => cf5d4648fde06cb520ace381e0be9853, NAME => 'TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:14:13,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => cf5d4648fde06cb520ace381e0be9853, NAME => 'TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:14:13,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cf5d4648fde06cb520ace381e0be9853, NAME => 'TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cf5d4648fde06cb520ace381e0be9853, NAME => 'TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:14:13,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-23T16:14:13,714 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:13,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59584 deadline: 1732378513708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:13,715 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:13,715 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:13,715 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:13,715 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:14:13,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59586 deadline: 1732378513708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:13,715 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:13,715 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:13,715 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:14:13,719 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:13,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59638 deadline: 1732378513718, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:13,719 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:13,719 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:13,720 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:13,720 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:14:13,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59614 deadline: 1732378513718, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:13,720 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:13,720 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:13,720 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:13,720 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:14:13,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59628 deadline: 1732378513718, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:13,721 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:13,721 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:13,721 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:14:13,845 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-23T16:14:13,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853. 2024-11-23T16:14:13,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853. as already flushing 2024-11-23T16:14:13,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853. 2024-11-23T16:14:13,845 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => cf5d4648fde06cb520ace381e0be9853, NAME => 'TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:14:13,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => cf5d4648fde06cb520ace381e0be9853, NAME => 'TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:14:13,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cf5d4648fde06cb520ace381e0be9853, NAME => 'TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cf5d4648fde06cb520ace381e0be9853, NAME => 'TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:14:13,874 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=53 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/.tmp/C/14c9ba67b3cd4082bdd8dab406b10fbb 2024-11-23T16:14:13,880 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/.tmp/A/a8c0de2db7aa45a8b430bae628f15ee0 as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/A/a8c0de2db7aa45a8b430bae628f15ee0 2024-11-23T16:14:13,886 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/A/a8c0de2db7aa45a8b430bae628f15ee0, entries=150, sequenceid=53, filesize=30.2 K 2024-11-23T16:14:13,887 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/.tmp/B/0eaab5d5995441b5aa72f557fd1ea17d as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/B/0eaab5d5995441b5aa72f557fd1ea17d 2024-11-23T16:14:13,892 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/B/0eaab5d5995441b5aa72f557fd1ea17d, entries=150, sequenceid=53, filesize=11.7 K 2024-11-23T16:14:13,893 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/.tmp/C/14c9ba67b3cd4082bdd8dab406b10fbb as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/C/14c9ba67b3cd4082bdd8dab406b10fbb 2024-11-23T16:14:13,898 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/C/14c9ba67b3cd4082bdd8dab406b10fbb, entries=150, sequenceid=53, filesize=11.7 K 2024-11-23T16:14:13,900 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~405.56 KB/415296, currentSize=40.25 KB/41220 for cf5d4648fde06cb520ace381e0be9853 in 531ms, sequenceid=53, compaction requested=false 2024-11-23T16:14:13,900 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cf5d4648fde06cb520ace381e0be9853: 2024-11-23T16:14:13,999 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-23T16:14:13,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853. 2024-11-23T16:14:13,999 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2902): Flushing cf5d4648fde06cb520ace381e0be9853 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-11-23T16:14:14,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf5d4648fde06cb520ace381e0be9853, store=A 2024-11-23T16:14:14,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-23T16:14:14,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf5d4648fde06cb520ace381e0be9853, store=B 2024-11-23T16:14:14,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-23T16:14:14,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf5d4648fde06cb520ace381e0be9853, store=C 2024-11-23T16:14:14,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-23T16:14:14,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112346db5400d0c94326ac5184a2f680320e_cf5d4648fde06cb520ace381e0be9853 is 50, key is test_row_0/A:col10/1732378453383/Put/seqid=0 2024-11-23T16:14:14,029 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073741956_1132 (size=12154) 2024-11-23T16:14:14,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:14,038 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112346db5400d0c94326ac5184a2f680320e_cf5d4648fde06cb520ace381e0be9853 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112346db5400d0c94326ac5184a2f680320e_cf5d4648fde06cb520ace381e0be9853 2024-11-23T16:14:14,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/.tmp/A/fd413a345eeb47718b7fd8958867c552, store: [table=TestAcidGuarantees family=A region=cf5d4648fde06cb520ace381e0be9853] 2024-11-23T16:14:14,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/.tmp/A/fd413a345eeb47718b7fd8958867c552 is 175, key is test_row_0/A:col10/1732378453383/Put/seqid=0 2024-11-23T16:14:14,046 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073741957_1133 (size=30955) 2024-11-23T16:14:14,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(8855): Flush requested on cf5d4648fde06cb520ace381e0be9853 2024-11-23T16:14:14,059 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853. as already flushing 2024-11-23T16:14:14,097 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1a7c5470b472:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1732378454072, max timestamp=1732378454093Num uniques 859; 2024-11-23T16:14:14,097 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1a7c5470b472:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-23T16:14:14,097 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1a7c5470b472:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1732378454072, max timestamp=1732378454093Num uniques 859; 2024-11-23T16:14:14,097 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1a7c5470b472:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-23T16:14:14,099 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1a7c5470b472:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732378454082, max timestamp=1732378454093Num uniques 859; 2024-11-23T16:14:14,100 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1a7c5470b472:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-23T16:14:14,124 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:14,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59614 deadline: 1732378514120, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:14,125 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:14,125 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:14,125 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:14:14,125 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:14,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59628 deadline: 1732378514121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:14,126 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:14,126 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:14,126 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:14:14,126 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:14,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59638 deadline: 1732378514122, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:14,126 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:14,126 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:14,126 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:14:14,132 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:14,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59584 deadline: 1732378514124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:14,133 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:14,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59586 deadline: 1732378514124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:14,133 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:14,134 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:14,134 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:14:14,134 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:14,134 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:14,134 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:14:14,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-23T16:14:14,236 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:14,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59614 deadline: 1732378514228, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:14,237 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:14,237 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:14,237 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:14:14,237 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:14,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59628 deadline: 1732378514228, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:14,238 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:14,238 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:14,238 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:14:14,239 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:14,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59638 deadline: 1732378514239, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:14,240 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:14,240 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:14,240 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:14:14,244 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:14,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59586 deadline: 1732378514239, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:14,245 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:14,245 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:14,245 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:14:14,245 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:14,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59584 deadline: 1732378514239, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:14,246 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:14,246 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:14,246 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:14:14,447 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=62, memsize=13.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/.tmp/A/fd413a345eeb47718b7fd8958867c552 2024-11-23T16:14:14,452 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:14,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59614 deadline: 1732378514448, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:14,453 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:14,453 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:14,453 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:14,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59638 deadline: 1732378514449, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:14,453 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:14:14,454 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:14,454 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:14,454 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:14,454 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:14:14,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59584 deadline: 1732378514449, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:14,454 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:14,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59628 deadline: 1732378514449, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:14,454 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:14,454 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:14,454 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:14:14,455 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:14,455 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:14,455 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:14,455 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:14:14,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59586 deadline: 1732378514449, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:14,455 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:14,455 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:14,455 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:14:14,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/.tmp/B/69dda82214634ca8a905b57d1914de79 is 50, key is test_row_0/B:col10/1732378453383/Put/seqid=0 2024-11-23T16:14:14,479 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073741958_1134 (size=12001) 2024-11-23T16:14:14,750 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-23T16:14:14,760 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:14,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59614 deadline: 1732378514758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:14,760 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:14,761 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:14,761 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:14,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59638 deadline: 1732378514758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:14,761 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:14:14,761 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:14,761 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:14,761 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:14,761 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:14:14,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59584 deadline: 1732378514758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:14,761 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:14,761 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:14,762 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:14,762 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:14:14,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59628 deadline: 1732378514758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:14,762 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:14,762 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:14,762 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:14:14,769 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:14,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59586 deadline: 1732378514769, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:14,778 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:14,778 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:14,778 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:14:14,880 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=62 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/.tmp/B/69dda82214634ca8a905b57d1914de79 2024-11-23T16:14:14,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/.tmp/C/3a50e69226a54b7b804803c663dc3c63 is 50, key is test_row_0/C:col10/1732378453383/Put/seqid=0 2024-11-23T16:14:14,898 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073741959_1135 (size=12001) 2024-11-23T16:14:15,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-23T16:14:15,271 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:15,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59614 deadline: 1732378515268, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:15,271 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:15,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59638 deadline: 1732378515268, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:15,272 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:15,272 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:15,272 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:15,272 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:14:15,272 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:15,272 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:15,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59584 deadline: 1732378515268, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:15,272 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:14:15,272 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:15,272 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:15,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59628 deadline: 1732378515269, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:15,272 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:15,272 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:14:15,273 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:15,273 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:15,273 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:14:15,288 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:15,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59586 deadline: 1732378515288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:15,289 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:15,289 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:15,289 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:14:15,299 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=62 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/.tmp/C/3a50e69226a54b7b804803c663dc3c63 2024-11-23T16:14:15,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/.tmp/A/fd413a345eeb47718b7fd8958867c552 as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/A/fd413a345eeb47718b7fd8958867c552 2024-11-23T16:14:15,312 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/A/fd413a345eeb47718b7fd8958867c552, entries=150, sequenceid=62, filesize=30.2 K 2024-11-23T16:14:15,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/.tmp/B/69dda82214634ca8a905b57d1914de79 as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/B/69dda82214634ca8a905b57d1914de79 2024-11-23T16:14:15,320 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/B/69dda82214634ca8a905b57d1914de79, entries=150, sequenceid=62, filesize=11.7 K 2024-11-23T16:14:15,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/.tmp/C/3a50e69226a54b7b804803c663dc3c63 as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/C/3a50e69226a54b7b804803c663dc3c63 2024-11-23T16:14:15,327 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/C/3a50e69226a54b7b804803c663dc3c63, entries=150, sequenceid=62, filesize=11.7 K 2024-11-23T16:14:15,329 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=140.89 KB/144270 for cf5d4648fde06cb520ace381e0be9853 in 1329ms, sequenceid=62, compaction requested=true 2024-11-23T16:14:15,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2603): Flush status journal for cf5d4648fde06cb520ace381e0be9853: 2024-11-23T16:14:15,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853. 2024-11-23T16:14:15,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=41 2024-11-23T16:14:15,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.HMaster(4169): Remote procedure done, pid=41 2024-11-23T16:14:15,333 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=41, resume processing ppid=40 2024-11-23T16:14:15,333 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=41, ppid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2580 sec 2024-11-23T16:14:15,335 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees in 2.2650 sec 2024-11-23T16:14:16,286 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cf5d4648fde06cb520ace381e0be9853 3/3 column families, dataSize=161.02 KB heapSize=456 KB 2024-11-23T16:14:16,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(8855): Flush requested on cf5d4648fde06cb520ace381e0be9853 2024-11-23T16:14:16,288 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf5d4648fde06cb520ace381e0be9853, store=A 2024-11-23T16:14:16,288 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-23T16:14:16,289 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf5d4648fde06cb520ace381e0be9853, store=B 2024-11-23T16:14:16,289 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-23T16:14:16,289 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf5d4648fde06cb520ace381e0be9853, store=C 2024-11-23T16:14:16,289 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-23T16:14:16,293 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:16,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59628 deadline: 1732378516289, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:16,293 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:16,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59584 deadline: 1732378516289, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:16,293 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:16,293 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:16,294 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:14:16,294 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:16,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59638 deadline: 1732378516291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:16,294 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:16,294 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:16,294 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:14:16,294 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:16,294 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:16,294 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:14:16,298 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241123c360bbed6efa42e783bbe9da99e487ca_cf5d4648fde06cb520ace381e0be9853 is 50, key is test_row_0/A:col10/1732378454121/Put/seqid=0 2024-11-23T16:14:16,299 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:16,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59614 deadline: 1732378516293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:16,299 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:16,299 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:16,300 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:14:16,302 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073741960_1136 (size=14594) 2024-11-23T16:14:16,305 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:16,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59586 deadline: 1732378516298, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:16,305 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 1a7c5470b472,44715,1732378420375 for 'test_row_2' in cf5d4648fde06cb520ace381e0be9853 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2181 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-23T16:14:16,306 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:16,306 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:16,306 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:14:16,403 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:16,404 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:16,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59628 deadline: 1732378516398, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:16,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59584 deadline: 1732378516398, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:16,404 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:16,404 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:16,404 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:16,404 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:14:16,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59638 deadline: 1732378516398, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:16,405 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:16,405 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:16,405 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:14:16,405 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:16,405 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:16,405 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:14:16,415 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:16,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59614 deadline: 1732378516408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:16,415 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:16,415 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:16,416 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:14:16,610 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:16,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59584 deadline: 1732378516608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:16,610 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:16,611 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:16,611 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:16,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59628 deadline: 1732378516608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:16,611 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:14:16,611 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:16,611 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:16,611 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:14:16,622 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:16,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59638 deadline: 1732378516618, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:16,622 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:16,622 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:16,622 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:14:16,632 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:16,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59614 deadline: 1732378516628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:16,632 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:16,632 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:16,632 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:14:16,703 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:16,708 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241123c360bbed6efa42e783bbe9da99e487ca_cf5d4648fde06cb520ace381e0be9853 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123c360bbed6efa42e783bbe9da99e487ca_cf5d4648fde06cb520ace381e0be9853 2024-11-23T16:14:16,709 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/.tmp/A/93c3613f1946487e90376f2ac7ba6b5b, store: [table=TestAcidGuarantees family=A region=cf5d4648fde06cb520ace381e0be9853] 2024-11-23T16:14:16,710 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/.tmp/A/93c3613f1946487e90376f2ac7ba6b5b is 175, key is test_row_0/A:col10/1732378454121/Put/seqid=0 2024-11-23T16:14:16,714 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073741961_1137 (size=39549) 2024-11-23T16:14:16,921 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:16,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59584 deadline: 1732378516917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:16,922 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:16,922 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:16,922 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:16,922 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:14:16,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59628 deadline: 1732378516918, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:16,922 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:16,922 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:16,923 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:14:16,928 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:16,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59638 deadline: 1732378516928, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:16,929 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:16,929 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:16,929 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:14:16,941 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:16,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59614 deadline: 1732378516938, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:16,942 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:16,942 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:16,942 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:14:17,115 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=102, memsize=58.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/.tmp/A/93c3613f1946487e90376f2ac7ba6b5b 2024-11-23T16:14:17,126 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/.tmp/B/a04ca98c2c964164a6d8632875d0846d is 50, key is test_row_0/B:col10/1732378454121/Put/seqid=0 2024-11-23T16:14:17,134 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073741962_1138 (size=12001) 2024-11-23T16:14:17,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-23T16:14:17,238 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-23T16:14:17,238 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-23T16:14:17,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] procedure2.ProcedureExecutor(1139): Stored pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees 2024-11-23T16:14:17,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-23T16:14:17,241 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-23T16:14:17,241 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-23T16:14:17,241 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=43, ppid=42, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-23T16:14:17,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-23T16:14:17,394 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-23T16:14:17,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853. 2024-11-23T16:14:17,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853. as already flushing 2024-11-23T16:14:17,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853. 2024-11-23T16:14:17,395 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => cf5d4648fde06cb520ace381e0be9853, NAME => 'TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:14:17,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => cf5d4648fde06cb520ace381e0be9853, NAME => 'TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:14:17,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cf5d4648fde06cb520ace381e0be9853, NAME => 'TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cf5d4648fde06cb520ace381e0be9853, NAME => 'TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:14:17,431 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:17,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59584 deadline: 1732378517428, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:17,432 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:17,432 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:17,432 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:14:17,446 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:17,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59628 deadline: 1732378517438, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:17,447 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:17,447 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:17,447 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:17,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59638 deadline: 1732378517439, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:17,447 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:14:17,447 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:17,447 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:17,447 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:14:17,449 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:17,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59614 deadline: 1732378517448, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:17,449 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:17,449 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:17,449 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:14:17,535 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=102 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/.tmp/B/a04ca98c2c964164a6d8632875d0846d 2024-11-23T16:14:17,545 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/.tmp/C/9c36aa9387174758a5218a7e1dfdf61e is 50, key is test_row_0/C:col10/1732378454121/Put/seqid=0 2024-11-23T16:14:17,548 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-23T16:14:17,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853. 2024-11-23T16:14:17,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853. as already flushing 2024-11-23T16:14:17,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853. 2024-11-23T16:14:17,549 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => cf5d4648fde06cb520ace381e0be9853, NAME => 'TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:14:17,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => cf5d4648fde06cb520ace381e0be9853, NAME => 'TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:14:17,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cf5d4648fde06cb520ace381e0be9853, NAME => 'TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cf5d4648fde06cb520ace381e0be9853, NAME => 'TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:14:17,552 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073741963_1139 (size=12001) 2024-11-23T16:14:17,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-23T16:14:17,702 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-23T16:14:17,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853. 2024-11-23T16:14:17,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853. as already flushing 2024-11-23T16:14:17,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853. 2024-11-23T16:14:17,702 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => cf5d4648fde06cb520ace381e0be9853, NAME => 'TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:14:17,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => cf5d4648fde06cb520ace381e0be9853, NAME => 'TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:14:17,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cf5d4648fde06cb520ace381e0be9853, NAME => 'TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cf5d4648fde06cb520ace381e0be9853, NAME => 'TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:14:17,855 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-23T16:14:17,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853. 2024-11-23T16:14:17,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853. as already flushing 2024-11-23T16:14:17,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853. 2024-11-23T16:14:17,856 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => cf5d4648fde06cb520ace381e0be9853, NAME => 'TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:14:17,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => cf5d4648fde06cb520ace381e0be9853, NAME => 'TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:14:17,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cf5d4648fde06cb520ace381e0be9853, NAME => 'TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cf5d4648fde06cb520ace381e0be9853, NAME => 'TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:14:17,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-23T16:14:17,953 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=102 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/.tmp/C/9c36aa9387174758a5218a7e1dfdf61e 2024-11-23T16:14:17,960 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/.tmp/A/93c3613f1946487e90376f2ac7ba6b5b as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/A/93c3613f1946487e90376f2ac7ba6b5b 2024-11-23T16:14:17,965 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/A/93c3613f1946487e90376f2ac7ba6b5b, entries=200, sequenceid=102, filesize=38.6 K 2024-11-23T16:14:17,966 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/.tmp/B/a04ca98c2c964164a6d8632875d0846d as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/B/a04ca98c2c964164a6d8632875d0846d 2024-11-23T16:14:17,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:17,971 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/B/a04ca98c2c964164a6d8632875d0846d, entries=150, sequenceid=102, filesize=11.7 K 2024-11-23T16:14:17,972 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/.tmp/C/9c36aa9387174758a5218a7e1dfdf61e as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/C/9c36aa9387174758a5218a7e1dfdf61e 2024-11-23T16:14:17,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:17,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:17,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:17,977 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/C/9c36aa9387174758a5218a7e1dfdf61e, entries=150, sequenceid=102, filesize=11.7 K 2024-11-23T16:14:17,978 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~167.72 KB/171750, heapSize ~475.88 KB/487296, currentSize=13.42 KB/13740 for cf5d4648fde06cb520ace381e0be9853 in 1692ms, sequenceid=102, compaction requested=true 2024-11-23T16:14:17,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:17,978 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cf5d4648fde06cb520ace381e0be9853: 2024-11-23T16:14:17,978 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cf5d4648fde06cb520ace381e0be9853:A, priority=-2147483648, current under compaction store size is 1 2024-11-23T16:14:17,978 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T16:14:17,978 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-23T16:14:17,978 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cf5d4648fde06cb520ace381e0be9853:B, priority=-2147483648, current under compaction store size is 2 2024-11-23T16:14:17,978 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T16:14:17,979 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-23T16:14:17,979 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cf5d4648fde06cb520ace381e0be9853:C, priority=-2147483648, current under compaction store size is 3 2024-11-23T16:14:17,979 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T16:14:17,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:17,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:17,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:17,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:17,981 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 158192 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-23T16:14:17,981 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HStore(1541): cf5d4648fde06cb520ace381e0be9853/A is initiating minor compaction (all files) 2024-11-23T16:14:17,981 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cf5d4648fde06cb520ace381e0be9853/A in TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853. 2024-11-23T16:14:17,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:17,982 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/A/389baa2344bd4e88a84af99676683d7c, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/A/a8c0de2db7aa45a8b430bae628f15ee0, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/A/fd413a345eeb47718b7fd8958867c552, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/A/93c3613f1946487e90376f2ac7ba6b5b] into tmpdir=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/.tmp, totalSize=154.5 K 2024-11-23T16:14:17,982 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853. 2024-11-23T16:14:17,982 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853. files: [hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/A/389baa2344bd4e88a84af99676683d7c, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/A/a8c0de2db7aa45a8b430bae628f15ee0, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/A/fd413a345eeb47718b7fd8958867c552, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/A/93c3613f1946487e90376f2ac7ba6b5b] 2024-11-23T16:14:17,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:17,982 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-23T16:14:17,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:17,982 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HStore(1541): cf5d4648fde06cb520ace381e0be9853/B is initiating minor compaction (all files) 2024-11-23T16:14:17,982 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cf5d4648fde06cb520ace381e0be9853/B in TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853. 2024-11-23T16:14:17,983 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/B/a4ba58cfe0284efa8306d9f0f422a16c, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/B/0eaab5d5995441b5aa72f557fd1ea17d, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/B/69dda82214634ca8a905b57d1914de79, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/B/a04ca98c2c964164a6d8632875d0846d] into tmpdir=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/.tmp, totalSize=46.9 K 2024-11-23T16:14:17,983 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.Compactor(225): Compacting a4ba58cfe0284efa8306d9f0f422a16c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1732378453083 2024-11-23T16:14:17,983 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.Compactor(225): Compacting 389baa2344bd4e88a84af99676683d7c, keycount=300, bloomtype=ROW, size=55.4 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1732378453083 2024-11-23T16:14:17,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:17,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:17,983 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.Compactor(225): Compacting 0eaab5d5995441b5aa72f557fd1ea17d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1732378453244 2024-11-23T16:14:17,984 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.Compactor(225): Compacting a8c0de2db7aa45a8b430bae628f15ee0, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1732378453244 2024-11-23T16:14:17,984 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.Compactor(225): Compacting 69dda82214634ca8a905b57d1914de79, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1732378453371 2024-11-23T16:14:17,984 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.Compactor(225): Compacting fd413a345eeb47718b7fd8958867c552, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1732378453371 2024-11-23T16:14:17,985 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.Compactor(225): Compacting a04ca98c2c964164a6d8632875d0846d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=102, earliestPutTs=1732378454110 2024-11-23T16:14:17,985 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.Compactor(225): Compacting 93c3613f1946487e90376f2ac7ba6b5b, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=102, earliestPutTs=1732378454110 2024-11-23T16:14:17,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:17,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:17,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:17,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:17,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:17,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:17,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:17,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:17,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:17,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:17,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:17,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:17,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:17,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:17,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:17,996 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cf5d4648fde06cb520ace381e0be9853#B#compaction#120 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T16:14:17,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:17,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:17,996 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/.tmp/B/82d4234caa5043b3b53db13b01f44556 is 50, key is test_row_0/B:col10/1732378454121/Put/seqid=0 2024-11-23T16:14:17,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:17,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:17,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:17,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:17,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:17,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,004 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=cf5d4648fde06cb520ace381e0be9853] 2024-11-23T16:14:18,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,009 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-23T16:14:18,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853. 2024-11-23T16:14:18,010 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2902): Flushing cf5d4648fde06cb520ace381e0be9853 3/3 column families, dataSize=13.42 KB heapSize=40.59 KB 2024-11-23T16:14:18,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf5d4648fde06cb520ace381e0be9853, store=A 2024-11-23T16:14:18,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-23T16:14:18,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf5d4648fde06cb520ace381e0be9853, store=B 2024-11-23T16:14:18,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-23T16:14:18,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf5d4648fde06cb520ace381e0be9853, store=C 2024-11-23T16:14:18,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-23T16:14:18,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,017 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241123064c1482b9704fa794262b60ae1d4398_cf5d4648fde06cb520ace381e0be9853 store=[table=TestAcidGuarantees family=A region=cf5d4648fde06cb520ace381e0be9853] 2024-11-23T16:14:18,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,022 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241123064c1482b9704fa794262b60ae1d4398_cf5d4648fde06cb520ace381e0be9853, store=[table=TestAcidGuarantees family=A region=cf5d4648fde06cb520ace381e0be9853] 2024-11-23T16:14:18,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,023 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241123064c1482b9704fa794262b60ae1d4398_cf5d4648fde06cb520ace381e0be9853 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=cf5d4648fde06cb520ace381e0be9853] 2024-11-23T16:14:18,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,047 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073741964_1140 (size=12139) 2024-11-23T16:14:18,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,054 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/.tmp/B/82d4234caa5043b3b53db13b01f44556 as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/B/82d4234caa5043b3b53db13b01f44556 2024-11-23T16:14:18,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241123e8d265743a374694b89f4e1209acaa82_cf5d4648fde06cb520ace381e0be9853 is 50, key is test_row_1/A:col10/1732378456290/Put/seqid=0 2024-11-23T16:14:18,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,063 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in cf5d4648fde06cb520ace381e0be9853/B of cf5d4648fde06cb520ace381e0be9853 into 82d4234caa5043b3b53db13b01f44556(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T16:14:18,063 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cf5d4648fde06cb520ace381e0be9853: 2024-11-23T16:14:18,063 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., storeName=cf5d4648fde06cb520ace381e0be9853/B, priority=12, startTime=1732378457978; duration=0sec 2024-11-23T16:14:18,064 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T16:14:18,064 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cf5d4648fde06cb520ace381e0be9853:B 2024-11-23T16:14:18,064 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-23T16:14:18,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,066 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-23T16:14:18,066 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HStore(1541): cf5d4648fde06cb520ace381e0be9853/C is initiating minor compaction (all files) 2024-11-23T16:14:18,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,066 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cf5d4648fde06cb520ace381e0be9853/C in TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853. 2024-11-23T16:14:18,066 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/C/5bf42799999d403d8173ae3cd6a75c07, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/C/14c9ba67b3cd4082bdd8dab406b10fbb, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/C/3a50e69226a54b7b804803c663dc3c63, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/C/9c36aa9387174758a5218a7e1dfdf61e] into tmpdir=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/.tmp, totalSize=46.9 K 2024-11-23T16:14:18,067 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.Compactor(225): Compacting 5bf42799999d403d8173ae3cd6a75c07, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1732378453083 2024-11-23T16:14:18,067 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.Compactor(225): Compacting 14c9ba67b3cd4082bdd8dab406b10fbb, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1732378453244 2024-11-23T16:14:18,068 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.Compactor(225): Compacting 3a50e69226a54b7b804803c663dc3c63, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1732378453371 2024-11-23T16:14:18,068 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.Compactor(225): Compacting 9c36aa9387174758a5218a7e1dfdf61e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=102, earliestPutTs=1732378454110 2024-11-23T16:14:18,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,069 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073741965_1141 (size=4469) 2024-11-23T16:14:18,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,078 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073741966_1142 (size=9714) 2024-11-23T16:14:18,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,081 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cf5d4648fde06cb520ace381e0be9853#C#compaction#123 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-23T16:14:18,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,082 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/.tmp/C/709f26037d9e498db94a1930f09e167b is 50, key is test_row_0/C:col10/1732378454121/Put/seqid=0 2024-11-23T16:14:18,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,086 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241123e8d265743a374694b89f4e1209acaa82_cf5d4648fde06cb520ace381e0be9853 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123e8d265743a374694b89f4e1209acaa82_cf5d4648fde06cb520ace381e0be9853 2024-11-23T16:14:18,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/.tmp/A/1a55f4f0a8e84edaba42163864b60e59, store: [table=TestAcidGuarantees family=A region=cf5d4648fde06cb520ace381e0be9853] 2024-11-23T16:14:18,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/.tmp/A/1a55f4f0a8e84edaba42163864b60e59 is 175, key is test_row_1/A:col10/1732378456290/Put/seqid=0 2024-11-23T16:14:18,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,100 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073741967_1143 (size=12139) 2024-11-23T16:14:18,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,105 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073741968_1144 (size=22361) 2024-11-23T16:14:18,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,106 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=107, memsize=4.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/.tmp/A/1a55f4f0a8e84edaba42163864b60e59 2024-11-23T16:14:18,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,112 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/.tmp/C/709f26037d9e498db94a1930f09e167b as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/C/709f26037d9e498db94a1930f09e167b 2024-11-23T16:14:18,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,120 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in cf5d4648fde06cb520ace381e0be9853/C of cf5d4648fde06cb520ace381e0be9853 into 709f26037d9e498db94a1930f09e167b(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T16:14:18,121 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cf5d4648fde06cb520ace381e0be9853: 2024-11-23T16:14:18,121 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., storeName=cf5d4648fde06cb520ace381e0be9853/C, priority=12, startTime=1732378457978; duration=0sec 2024-11-23T16:14:18,121 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T16:14:18,121 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cf5d4648fde06cb520ace381e0be9853:C 2024-11-23T16:14:18,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/.tmp/B/1d598d0fa2594f62a6ff85d4131f2702 is 50, key is test_row_1/B:col10/1732378456290/Put/seqid=0 2024-11-23T16:14:18,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,142 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073741969_1145 (size=9657) 2024-11-23T16:14:18,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,143 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=107 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/.tmp/B/1d598d0fa2594f62a6ff85d4131f2702 2024-11-23T16:14:18,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/.tmp/C/d12fb284d6c142db95dd0da11d15d56b is 50, key is test_row_1/C:col10/1732378456290/Put/seqid=0 2024-11-23T16:14:18,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,156 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073741970_1146 (size=9657) 2024-11-23T16:14:18,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,156 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=107 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/.tmp/C/d12fb284d6c142db95dd0da11d15d56b 2024-11-23T16:14:18,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/.tmp/A/1a55f4f0a8e84edaba42163864b60e59 as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/A/1a55f4f0a8e84edaba42163864b60e59 2024-11-23T16:14:18,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,169 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/A/1a55f4f0a8e84edaba42163864b60e59, entries=100, sequenceid=107, filesize=21.8 K 2024-11-23T16:14:18,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/.tmp/B/1d598d0fa2594f62a6ff85d4131f2702 as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/B/1d598d0fa2594f62a6ff85d4131f2702 2024-11-23T16:14:18,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,177 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/B/1d598d0fa2594f62a6ff85d4131f2702, entries=100, sequenceid=107, filesize=9.4 K 2024-11-23T16:14:18,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/.tmp/C/d12fb284d6c142db95dd0da11d15d56b as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/C/d12fb284d6c142db95dd0da11d15d56b 2024-11-23T16:14:18,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,185 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/C/d12fb284d6c142db95dd0da11d15d56b, entries=100, sequenceid=107, filesize=9.4 K 2024-11-23T16:14:18,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,186 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(3140): Finished flush of dataSize ~13.42 KB/13740, heapSize ~40.55 KB/41520, currentSize=0 B/0 for cf5d4648fde06cb520ace381e0be9853 in 176ms, sequenceid=107, compaction requested=false 2024-11-23T16:14:18,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2603): Flush status journal for cf5d4648fde06cb520ace381e0be9853: 2024-11-23T16:14:18,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853. 2024-11-23T16:14:18,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=43 2024-11-23T16:14:18,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.HMaster(4169): Remote procedure done, pid=43 2024-11-23T16:14:18,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,191 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=43, resume processing ppid=42 2024-11-23T16:14:18,191 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, ppid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 947 msec 2024-11-23T16:14:18,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,193 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees in 954 msec 2024-11-23T16:14:18,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-23T16:14:18,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,378 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-23T16:14:18,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,379 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-23T16:14:18,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] procedure2.ProcedureExecutor(1139): Stored pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees 2024-11-23T16:14:18,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-23T16:14:18,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,382 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-23T16:14:18,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,382 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-23T16:14:18,383 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=45, ppid=44, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-23T16:14:18,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,473 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cf5d4648fde06cb520ace381e0be9853 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-23T16:14:18,473 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf5d4648fde06cb520ace381e0be9853, store=A 2024-11-23T16:14:18,473 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-23T16:14:18,473 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf5d4648fde06cb520ace381e0be9853, store=B 2024-11-23T16:14:18,473 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-23T16:14:18,473 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf5d4648fde06cb520ace381e0be9853, store=C 2024-11-23T16:14:18,473 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): cf5d4648fde06cb520ace381e0be9853#A#compaction#121 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T16:14:18,473 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-23T16:14:18,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,474 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/.tmp/A/dc257c88d995485a824a51457199c7f1 is 175, key is test_row_0/A:col10/1732378454121/Put/seqid=0 2024-11-23T16:14:18,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(8855): Flush requested on cf5d4648fde06cb520ace381e0be9853 2024-11-23T16:14:18,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,487 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073741971_1147 (size=31093) 2024-11-23T16:14:18,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-23T16:14:18,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,497 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/.tmp/A/dc257c88d995485a824a51457199c7f1 as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/A/dc257c88d995485a824a51457199c7f1 2024-11-23T16:14:18,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,497 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411239e1dd22508334809b5d7df8dc91e5699_cf5d4648fde06cb520ace381e0be9853 is 50, key is test_row_0/A:col10/1732378458469/Put/seqid=0 2024-11-23T16:14:18,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,509 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in cf5d4648fde06cb520ace381e0be9853/A of cf5d4648fde06cb520ace381e0be9853 into dc257c88d995485a824a51457199c7f1(size=30.4 K), total size for store is 52.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T16:14:18,509 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cf5d4648fde06cb520ace381e0be9853: 2024-11-23T16:14:18,509 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., storeName=cf5d4648fde06cb520ace381e0be9853/A, priority=12, startTime=1732378457978; duration=0sec 2024-11-23T16:14:18,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,509 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T16:14:18,509 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cf5d4648fde06cb520ace381e0be9853:A 2024-11-23T16:14:18,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,513 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073741972_1148 (size=17034) 2024-11-23T16:14:18,515 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:18,523 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411239e1dd22508334809b5d7df8dc91e5699_cf5d4648fde06cb520ace381e0be9853 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411239e1dd22508334809b5d7df8dc91e5699_cf5d4648fde06cb520ace381e0be9853 2024-11-23T16:14:18,524 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/.tmp/A/965712ddc1ff49dfb4706f98e3226a4b, store: [table=TestAcidGuarantees family=A region=cf5d4648fde06cb520ace381e0be9853] 2024-11-23T16:14:18,525 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/.tmp/A/965712ddc1ff49dfb4706f98e3226a4b is 175, key is test_row_0/A:col10/1732378458469/Put/seqid=0 2024-11-23T16:14:18,536 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1a7c5470b472:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=559, cellSize=25595, totalHeapSize=58376, min timestamp=1732378458499, max timestamp=1732378458533Num uniques 859; 2024-11-23T16:14:18,536 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1a7c5470b472:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=25595, suffix heap size=117064,new segment heap size=58376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=559 2024-11-23T16:14:18,536 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-23T16:14:18,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853. 2024-11-23T16:14:18,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853. as already flushing 2024-11-23T16:14:18,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853. 2024-11-23T16:14:18,537 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => cf5d4648fde06cb520ace381e0be9853, NAME => 'TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:14:18,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => cf5d4648fde06cb520ace381e0be9853, NAME => 'TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:14:18,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cf5d4648fde06cb520ace381e0be9853, NAME => 'TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cf5d4648fde06cb520ace381e0be9853, NAME => 'TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:14:18,539 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1a7c5470b472:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=559, cellSize=25595, totalHeapSize=58376, min timestamp=1732378458499, max timestamp=1732378458533Num uniques 859; 2024-11-23T16:14:18,539 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1a7c5470b472:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=25595, suffix heap size=117064,new segment heap size=58376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=559 2024-11-23T16:14:18,540 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1a7c5470b472:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=559, cellSize=25595, totalHeapSize=58376, min timestamp=1732378458499, max timestamp=1732378458533Num uniques 859; 2024-11-23T16:14:18,540 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1a7c5470b472:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=25595, suffix heap size=117064,new segment heap size=58376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=559 2024-11-23T16:14:18,548 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073741973_1149 (size=48135) 2024-11-23T16:14:18,550 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=119, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/.tmp/A/965712ddc1ff49dfb4706f98e3226a4b 2024-11-23T16:14:18,559 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/.tmp/B/d764ff88804e4a3f906461d54b49dddb is 50, key is test_row_0/B:col10/1732378458469/Put/seqid=0 2024-11-23T16:14:18,564 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:18,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59586 deadline: 1732378518553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:18,564 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:18,564 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:18,564 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:18,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59638 deadline: 1732378518561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:18,565 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:14:18,565 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:18,565 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:18,565 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:14:18,570 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:18,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59628 deadline: 1732378518563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:18,571 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073741974_1150 (size=9657) 2024-11-23T16:14:18,571 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:18,571 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:18,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59584 deadline: 1732378518564, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:18,571 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:18,571 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:14:18,571 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:18,572 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:18,572 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:18,572 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:14:18,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59614 deadline: 1732378518565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:18,572 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:18,572 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:18,572 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:14:18,670 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:18,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59586 deadline: 1732378518667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:18,670 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:18,670 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:18,670 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:18,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59638 deadline: 1732378518668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:18,670 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:14:18,671 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:18,671 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:18,671 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:14:18,680 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:18,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59628 deadline: 1732378518678, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:18,681 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:18,681 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:18,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59584 deadline: 1732378518678, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:18,681 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:18,681 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:14:18,681 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:18,681 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:18,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59614 deadline: 1732378518678, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:18,681 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:18,681 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:14:18,681 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:18,681 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:18,682 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:14:18,690 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-23T16:14:18,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853. 2024-11-23T16:14:18,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853. as already flushing 2024-11-23T16:14:18,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853. 2024-11-23T16:14:18,690 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => cf5d4648fde06cb520ace381e0be9853, NAME => 'TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:14:18,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => cf5d4648fde06cb520ace381e0be9853, NAME => 'TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:14:18,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cf5d4648fde06cb520ace381e0be9853, NAME => 'TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cf5d4648fde06cb520ace381e0be9853, NAME => 'TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:14:18,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-23T16:14:18,844 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-23T16:14:18,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853. 2024-11-23T16:14:18,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853. as already flushing 2024-11-23T16:14:18,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853. 2024-11-23T16:14:18,844 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => cf5d4648fde06cb520ace381e0be9853, NAME => 'TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:14:18,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => cf5d4648fde06cb520ace381e0be9853, NAME => 'TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:14:18,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cf5d4648fde06cb520ace381e0be9853, NAME => 'TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cf5d4648fde06cb520ace381e0be9853, NAME => 'TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:14:18,881 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:18,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59586 deadline: 1732378518878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:18,882 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:18,882 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:18,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59638 deadline: 1732378518878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:18,882 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:18,882 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:14:18,883 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:18,883 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:18,883 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:14:18,891 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:18,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59628 deadline: 1732378518888, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:18,891 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:18,891 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:18,891 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:18,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59614 deadline: 1732378518889, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:18,892 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:14:18,892 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:18,892 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:18,892 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:18,892 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:14:18,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59584 deadline: 1732378518889, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:18,893 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:18,893 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:18,893 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:14:18,972 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=119 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/.tmp/B/d764ff88804e4a3f906461d54b49dddb 2024-11-23T16:14:18,998 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/.tmp/C/2783233a743c4d6bbb2221923dc24c3e is 50, key is test_row_0/C:col10/1732378458469/Put/seqid=0 2024-11-23T16:14:18,999 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-23T16:14:18,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853. 2024-11-23T16:14:18,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853. as already flushing 2024-11-23T16:14:18,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853. 2024-11-23T16:14:18,999 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => cf5d4648fde06cb520ace381e0be9853, NAME => 'TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:14:18,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => cf5d4648fde06cb520ace381e0be9853, NAME => 'TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:14:19,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cf5d4648fde06cb520ace381e0be9853, NAME => 'TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cf5d4648fde06cb520ace381e0be9853, NAME => 'TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:14:19,006 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073741975_1151 (size=9657) 2024-11-23T16:14:19,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-23T16:14:19,153 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-23T16:14:19,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853. 2024-11-23T16:14:19,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853. as already flushing 2024-11-23T16:14:19,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853. 2024-11-23T16:14:19,153 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => cf5d4648fde06cb520ace381e0be9853, NAME => 'TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:14:19,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => cf5d4648fde06cb520ace381e0be9853, NAME => 'TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:14:19,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cf5d4648fde06cb520ace381e0be9853, NAME => 'TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cf5d4648fde06cb520ace381e0be9853, NAME => 'TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:14:19,190 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:19,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59586 deadline: 1732378519188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:19,190 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:19,190 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:19,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59638 deadline: 1732378519188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:19,190 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:19,190 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:14:19,191 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:19,191 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:19,191 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:14:19,201 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:19,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59628 deadline: 1732378519198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:19,201 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:19,201 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:19,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59614 deadline: 1732378519198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:19,202 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:19,202 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:14:19,202 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:19,202 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:19,202 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:19,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59584 deadline: 1732378519198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:19,202 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:14:19,202 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:19,202 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:19,202 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:14:19,306 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-23T16:14:19,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853. 2024-11-23T16:14:19,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853. as already flushing 2024-11-23T16:14:19,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853. 2024-11-23T16:14:19,307 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => cf5d4648fde06cb520ace381e0be9853, NAME => 'TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:14:19,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => cf5d4648fde06cb520ace381e0be9853, NAME => 'TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:14:19,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cf5d4648fde06cb520ace381e0be9853, NAME => 'TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cf5d4648fde06cb520ace381e0be9853, NAME => 'TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:14:19,407 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=119 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/.tmp/C/2783233a743c4d6bbb2221923dc24c3e 2024-11-23T16:14:19,414 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/.tmp/A/965712ddc1ff49dfb4706f98e3226a4b as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/A/965712ddc1ff49dfb4706f98e3226a4b 2024-11-23T16:14:19,418 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/A/965712ddc1ff49dfb4706f98e3226a4b, entries=250, sequenceid=119, filesize=47.0 K 2024-11-23T16:14:19,419 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/.tmp/B/d764ff88804e4a3f906461d54b49dddb as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/B/d764ff88804e4a3f906461d54b49dddb 2024-11-23T16:14:19,425 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/B/d764ff88804e4a3f906461d54b49dddb, entries=100, sequenceid=119, filesize=9.4 K 2024-11-23T16:14:19,426 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/.tmp/C/2783233a743c4d6bbb2221923dc24c3e as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/C/2783233a743c4d6bbb2221923dc24c3e 2024-11-23T16:14:19,431 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/C/2783233a743c4d6bbb2221923dc24c3e, entries=100, sequenceid=119, filesize=9.4 K 2024-11-23T16:14:19,433 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=154.31 KB/158010 for cf5d4648fde06cb520ace381e0be9853 in 960ms, sequenceid=119, compaction requested=true 2024-11-23T16:14:19,433 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cf5d4648fde06cb520ace381e0be9853: 2024-11-23T16:14:19,434 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T16:14:19,434 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cf5d4648fde06cb520ace381e0be9853:A, priority=-2147483648, current under compaction store size is 1 2024-11-23T16:14:19,434 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T16:14:19,434 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cf5d4648fde06cb520ace381e0be9853:B, priority=-2147483648, current under compaction store size is 2 2024-11-23T16:14:19,434 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T16:14:19,434 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cf5d4648fde06cb520ace381e0be9853:C, priority=-2147483648, current under compaction store size is 3 2024-11-23T16:14:19,434 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T16:14:19,434 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T16:14:19,435 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101589 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T16:14:19,435 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HStore(1541): cf5d4648fde06cb520ace381e0be9853/A is initiating minor compaction (all files) 2024-11-23T16:14:19,435 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cf5d4648fde06cb520ace381e0be9853/A in TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853. 2024-11-23T16:14:19,435 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/A/dc257c88d995485a824a51457199c7f1, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/A/1a55f4f0a8e84edaba42163864b60e59, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/A/965712ddc1ff49dfb4706f98e3226a4b] into tmpdir=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/.tmp, totalSize=99.2 K 2024-11-23T16:14:19,435 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853. 2024-11-23T16:14:19,435 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853. files: [hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/A/dc257c88d995485a824a51457199c7f1, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/A/1a55f4f0a8e84edaba42163864b60e59, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/A/965712ddc1ff49dfb4706f98e3226a4b] 2024-11-23T16:14:19,435 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 31453 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T16:14:19,436 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HStore(1541): cf5d4648fde06cb520ace381e0be9853/B is initiating minor compaction (all files) 2024-11-23T16:14:19,436 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cf5d4648fde06cb520ace381e0be9853/B in TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853. 2024-11-23T16:14:19,436 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/B/82d4234caa5043b3b53db13b01f44556, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/B/1d598d0fa2594f62a6ff85d4131f2702, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/B/d764ff88804e4a3f906461d54b49dddb] into tmpdir=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/.tmp, totalSize=30.7 K 2024-11-23T16:14:19,436 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.Compactor(225): Compacting dc257c88d995485a824a51457199c7f1, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=102, earliestPutTs=1732378454110 2024-11-23T16:14:19,436 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1a55f4f0a8e84edaba42163864b60e59, keycount=100, bloomtype=ROW, size=21.8 K, encoding=NONE, compression=NONE, seqNum=107, earliestPutTs=1732378456289 2024-11-23T16:14:19,437 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.Compactor(225): Compacting 82d4234caa5043b3b53db13b01f44556, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=102, earliestPutTs=1732378454110 2024-11-23T16:14:19,437 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.Compactor(225): Compacting 965712ddc1ff49dfb4706f98e3226a4b, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1732378458421 2024-11-23T16:14:19,437 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.Compactor(225): Compacting 1d598d0fa2594f62a6ff85d4131f2702, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=107, earliestPutTs=1732378456289 2024-11-23T16:14:19,438 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.Compactor(225): Compacting d764ff88804e4a3f906461d54b49dddb, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1732378458456 2024-11-23T16:14:19,446 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=cf5d4648fde06cb520ace381e0be9853] 2024-11-23T16:14:19,448 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cf5d4648fde06cb520ace381e0be9853#B#compaction#130 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-23T16:14:19,449 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/.tmp/B/47365aaa2d7e4933b4b0be9429a13d88 is 50, key is test_row_0/B:col10/1732378458469/Put/seqid=0 2024-11-23T16:14:19,457 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024112304fca60679164be0bacc1172346a419d_cf5d4648fde06cb520ace381e0be9853 store=[table=TestAcidGuarantees family=A region=cf5d4648fde06cb520ace381e0be9853] 2024-11-23T16:14:19,459 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024112304fca60679164be0bacc1172346a419d_cf5d4648fde06cb520ace381e0be9853, store=[table=TestAcidGuarantees family=A region=cf5d4648fde06cb520ace381e0be9853] 2024-11-23T16:14:19,459 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112304fca60679164be0bacc1172346a419d_cf5d4648fde06cb520ace381e0be9853 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=cf5d4648fde06cb520ace381e0be9853] 2024-11-23T16:14:19,460 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-23T16:14:19,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853. 2024-11-23T16:14:19,460 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2902): Flushing cf5d4648fde06cb520ace381e0be9853 3/3 column families, dataSize=154.31 KB heapSize=407.25 KB 2024-11-23T16:14:19,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf5d4648fde06cb520ace381e0be9853, store=A 2024-11-23T16:14:19,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-23T16:14:19,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf5d4648fde06cb520ace381e0be9853, store=B 2024-11-23T16:14:19,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-23T16:14:19,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf5d4648fde06cb520ace381e0be9853, store=C 2024-11-23T16:14:19,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-23T16:14:19,462 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073741976_1152 (size=12241) 2024-11-23T16:14:19,466 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073741977_1153 (size=4469) 2024-11-23T16:14:19,469 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/.tmp/B/47365aaa2d7e4933b4b0be9429a13d88 as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/B/47365aaa2d7e4933b4b0be9429a13d88 2024-11-23T16:14:19,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112396b76e6ebded44d79dbddef7e93f2e94_cf5d4648fde06cb520ace381e0be9853 is 50, key is test_row_0/A:col10/1732378458561/Put/seqid=0 2024-11-23T16:14:19,475 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cf5d4648fde06cb520ace381e0be9853/B of cf5d4648fde06cb520ace381e0be9853 into 47365aaa2d7e4933b4b0be9429a13d88(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T16:14:19,475 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cf5d4648fde06cb520ace381e0be9853: 2024-11-23T16:14:19,475 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., storeName=cf5d4648fde06cb520ace381e0be9853/B, priority=13, startTime=1732378459434; duration=0sec 2024-11-23T16:14:19,476 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073741978_1154 (size=12304) 2024-11-23T16:14:19,476 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T16:14:19,476 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cf5d4648fde06cb520ace381e0be9853:B 2024-11-23T16:14:19,476 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T16:14:19,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:19,478 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 31453 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T16:14:19,478 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HStore(1541): cf5d4648fde06cb520ace381e0be9853/C is initiating minor compaction (all files) 2024-11-23T16:14:19,478 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cf5d4648fde06cb520ace381e0be9853/C in TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853. 2024-11-23T16:14:19,478 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/C/709f26037d9e498db94a1930f09e167b, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/C/d12fb284d6c142db95dd0da11d15d56b, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/C/2783233a743c4d6bbb2221923dc24c3e] into tmpdir=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/.tmp, totalSize=30.7 K 2024-11-23T16:14:19,479 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.Compactor(225): Compacting 709f26037d9e498db94a1930f09e167b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=102, earliestPutTs=1732378454110 2024-11-23T16:14:19,479 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.Compactor(225): Compacting d12fb284d6c142db95dd0da11d15d56b, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=107, earliestPutTs=1732378456289 2024-11-23T16:14:19,480 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.Compactor(225): Compacting 2783233a743c4d6bbb2221923dc24c3e, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1732378458456 2024-11-23T16:14:19,481 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112396b76e6ebded44d79dbddef7e93f2e94_cf5d4648fde06cb520ace381e0be9853 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112396b76e6ebded44d79dbddef7e93f2e94_cf5d4648fde06cb520ace381e0be9853 2024-11-23T16:14:19,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/.tmp/A/fb2a54a1f0c640ea8f92799b615c30e9, store: [table=TestAcidGuarantees family=A region=cf5d4648fde06cb520ace381e0be9853] 2024-11-23T16:14:19,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/.tmp/A/fb2a54a1f0c640ea8f92799b615c30e9 is 175, key is test_row_0/A:col10/1732378458561/Put/seqid=0 2024-11-23T16:14:19,487 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073741979_1155 (size=31105) 2024-11-23T16:14:19,490 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cf5d4648fde06cb520ace381e0be9853#C#compaction#132 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-23T16:14:19,491 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/.tmp/C/cf130f6f9a254c50a6b9c4ebad74e11f is 50, key is test_row_0/C:col10/1732378458469/Put/seqid=0 2024-11-23T16:14:19,494 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073741980_1156 (size=12241) 2024-11-23T16:14:19,500 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/.tmp/C/cf130f6f9a254c50a6b9c4ebad74e11f as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/C/cf130f6f9a254c50a6b9c4ebad74e11f 2024-11-23T16:14:19,505 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cf5d4648fde06cb520ace381e0be9853/C of cf5d4648fde06cb520ace381e0be9853 into cf130f6f9a254c50a6b9c4ebad74e11f(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T16:14:19,505 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cf5d4648fde06cb520ace381e0be9853: 2024-11-23T16:14:19,505 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., storeName=cf5d4648fde06cb520ace381e0be9853/C, priority=13, startTime=1732378459434; duration=0sec 2024-11-23T16:14:19,505 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T16:14:19,505 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cf5d4648fde06cb520ace381e0be9853:C 2024-11-23T16:14:19,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-23T16:14:19,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(8855): Flush requested on cf5d4648fde06cb520ace381e0be9853 2024-11-23T16:14:19,703 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853. as already flushing 2024-11-23T16:14:19,711 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:19,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59584 deadline: 1732378519708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:19,712 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:19,712 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:19,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59586 deadline: 1732378519708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:19,712 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:19,712 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:14:19,712 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:19,712 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:19,712 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:19,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59638 deadline: 1732378519709, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:19,712 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:14:19,713 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:19,713 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:19,713 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:14:19,716 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:19,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59614 deadline: 1732378519710, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:19,717 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:19,717 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:19,717 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:19,717 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:14:19,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59628 deadline: 1732378519711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:19,717 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:19,717 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:19,717 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:14:19,822 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:19,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59638 deadline: 1732378519820, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:19,823 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:19,823 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:19,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59586 deadline: 1732378519820, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:19,823 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:19,823 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:14:19,823 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:19,824 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:19,824 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:14:19,830 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:19,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59614 deadline: 1732378519828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:19,830 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:19,830 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:19,831 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:19,831 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:14:19,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59628 deadline: 1732378519828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:19,831 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:19,831 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:19,831 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:14:19,868 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): cf5d4648fde06cb520ace381e0be9853#A#compaction#129 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T16:14:19,869 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/.tmp/A/90db0d62c1304f1f92806b95265a9106 is 175, key is test_row_0/A:col10/1732378458469/Put/seqid=0 2024-11-23T16:14:19,876 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073741981_1157 (size=31302) 2024-11-23T16:14:19,883 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/.tmp/A/90db0d62c1304f1f92806b95265a9106 as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/A/90db0d62c1304f1f92806b95265a9106 2024-11-23T16:14:19,888 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=152, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/.tmp/A/fb2a54a1f0c640ea8f92799b615c30e9 2024-11-23T16:14:19,892 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cf5d4648fde06cb520ace381e0be9853/A of cf5d4648fde06cb520ace381e0be9853 into 90db0d62c1304f1f92806b95265a9106(size=30.6 K), total size for store is 30.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T16:14:19,892 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cf5d4648fde06cb520ace381e0be9853: 2024-11-23T16:14:19,895 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., storeName=cf5d4648fde06cb520ace381e0be9853/A, priority=13, startTime=1732378459433; duration=0sec 2024-11-23T16:14:19,895 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T16:14:19,895 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cf5d4648fde06cb520ace381e0be9853:A 2024-11-23T16:14:19,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/.tmp/B/0718a41c649e459a8be955af356e718c is 50, key is test_row_0/B:col10/1732378458561/Put/seqid=0 2024-11-23T16:14:19,904 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073741982_1158 (size=12151) 2024-11-23T16:14:19,904 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=152 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/.tmp/B/0718a41c649e459a8be955af356e718c 2024-11-23T16:14:19,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/.tmp/C/7ded1e012fcf4760a8d6c12379c832f3 is 50, key is test_row_0/C:col10/1732378458561/Put/seqid=0 2024-11-23T16:14:19,919 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073741983_1159 (size=12151) 2024-11-23T16:14:20,032 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:20,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59638 deadline: 1732378520028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:20,033 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:20,033 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:20,033 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:20,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59586 deadline: 1732378520028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:20,033 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:14:20,033 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:20,033 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:20,033 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:14:20,043 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:20,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59614 deadline: 1732378520038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:20,043 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:20,043 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:20,043 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:20,043 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:14:20,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59628 deadline: 1732378520038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:20,044 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:20,044 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:20,044 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:14:20,108 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-23T16:14:20,108 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-23T16:14:20,320 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=152 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/.tmp/C/7ded1e012fcf4760a8d6c12379c832f3 2024-11-23T16:14:20,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/.tmp/A/fb2a54a1f0c640ea8f92799b615c30e9 as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/A/fb2a54a1f0c640ea8f92799b615c30e9 2024-11-23T16:14:20,334 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/A/fb2a54a1f0c640ea8f92799b615c30e9, entries=150, sequenceid=152, filesize=30.4 K 2024-11-23T16:14:20,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/.tmp/B/0718a41c649e459a8be955af356e718c as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/B/0718a41c649e459a8be955af356e718c 2024-11-23T16:14:20,340 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/B/0718a41c649e459a8be955af356e718c, entries=150, sequenceid=152, filesize=11.9 K 2024-11-23T16:14:20,340 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:20,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59586 deadline: 1732378520339, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:20,341 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:20,341 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:20,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59638 deadline: 1732378520339, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:20,341 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:20,341 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:14:20,341 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:20,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/.tmp/C/7ded1e012fcf4760a8d6c12379c832f3 as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/C/7ded1e012fcf4760a8d6c12379c832f3 2024-11-23T16:14:20,342 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:20,342 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:14:20,347 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/C/7ded1e012fcf4760a8d6c12379c832f3, entries=150, sequenceid=152, filesize=11.9 K 2024-11-23T16:14:20,348 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~407.20 KB/416976, currentSize=40.25 KB/41220 for cf5d4648fde06cb520ace381e0be9853 in 888ms, sequenceid=152, compaction requested=false 2024-11-23T16:14:20,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2603): Flush status journal for cf5d4648fde06cb520ace381e0be9853: 2024-11-23T16:14:20,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853. 2024-11-23T16:14:20,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=45 2024-11-23T16:14:20,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.HMaster(4169): Remote procedure done, pid=45 2024-11-23T16:14:20,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(8855): Flush requested on cf5d4648fde06cb520ace381e0be9853 2024-11-23T16:14:20,351 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cf5d4648fde06cb520ace381e0be9853 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-23T16:14:20,351 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf5d4648fde06cb520ace381e0be9853, store=A 2024-11-23T16:14:20,351 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-23T16:14:20,351 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf5d4648fde06cb520ace381e0be9853, store=B 2024-11-23T16:14:20,351 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-23T16:14:20,351 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf5d4648fde06cb520ace381e0be9853, store=C 2024-11-23T16:14:20,352 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-23T16:14:20,353 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=45, resume processing ppid=44 2024-11-23T16:14:20,353 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, ppid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9680 sec 2024-11-23T16:14:20,355 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees in 1.9750 sec 2024-11-23T16:14:20,367 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241123475a609c00be404e812c6939ed83f7fb_cf5d4648fde06cb520ace381e0be9853 is 50, key is test_row_0/A:col10/1732378460350/Put/seqid=0 2024-11-23T16:14:20,391 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073741984_1160 (size=14794) 2024-11-23T16:14:20,392 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:20,400 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241123475a609c00be404e812c6939ed83f7fb_cf5d4648fde06cb520ace381e0be9853 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123475a609c00be404e812c6939ed83f7fb_cf5d4648fde06cb520ace381e0be9853 2024-11-23T16:14:20,401 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/.tmp/A/274abfa78d934cd49d56bc1cc3ac58fd, store: [table=TestAcidGuarantees family=A region=cf5d4648fde06cb520ace381e0be9853] 2024-11-23T16:14:20,402 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/.tmp/A/274abfa78d934cd49d56bc1cc3ac58fd is 175, key is test_row_0/A:col10/1732378460350/Put/seqid=0 2024-11-23T16:14:20,414 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1a7c5470b472:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1732378460384, max timestamp=1732378460410Num uniques 859; 2024-11-23T16:14:20,414 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1a7c5470b472:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-11-23T16:14:20,415 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1a7c5470b472:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1732378460384, max timestamp=1732378460410Num uniques 859; 2024-11-23T16:14:20,415 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1a7c5470b472:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-11-23T16:14:20,415 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1a7c5470b472:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1732378460384, max timestamp=1732378460410Num uniques 859; 2024-11-23T16:14:20,415 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1a7c5470b472:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-11-23T16:14:20,419 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073741985_1161 (size=39749) 2024-11-23T16:14:20,458 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:20,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59614 deadline: 1732378520453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:20,458 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:20,458 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:20,459 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:14:20,462 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:20,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59628 deadline: 1732378520458, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:20,462 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:20,462 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:20,462 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:14:20,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-23T16:14:20,539 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-23T16:14:20,539 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-23T16:14:20,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] procedure2.ProcedureExecutor(1139): Stored pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees 2024-11-23T16:14:20,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-23T16:14:20,542 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-23T16:14:20,542 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-23T16:14:20,542 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=47, ppid=46, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-23T16:14:20,571 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:20,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59614 deadline: 1732378520568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:20,572 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:20,572 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:20,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59628 deadline: 1732378520568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:20,572 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:20,572 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:14:20,572 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:20,572 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:20,572 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:14:20,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-23T16:14:20,695 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-23T16:14:20,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853. 2024-11-23T16:14:20,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853. as already flushing 2024-11-23T16:14:20,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853. 2024-11-23T16:14:20,696 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => cf5d4648fde06cb520ace381e0be9853, NAME => 'TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:14:20,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => cf5d4648fde06cb520ace381e0be9853, NAME => 'TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:14:20,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cf5d4648fde06cb520ace381e0be9853, NAME => 'TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cf5d4648fde06cb520ace381e0be9853, NAME => 'TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:14:20,724 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:20,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59584 deadline: 1732378520718, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:20,725 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to 1a7c5470b472,44715,1732378420375 for 'test_row_1' in cf5d4648fde06cb520ace381e0be9853 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2161 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-23T16:14:20,726 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:20,726 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:20,726 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:14:20,781 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:20,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59614 deadline: 1732378520778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:20,781 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:20,781 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:20,781 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:20,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59628 deadline: 1732378520778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:20,781 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:14:20,782 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:20,782 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:20,783 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:14:20,820 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=165, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/.tmp/A/274abfa78d934cd49d56bc1cc3ac58fd 2024-11-23T16:14:20,839 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/.tmp/B/5cfcd5ab3a6249a7ac4da7a37cab896c is 50, key is test_row_0/B:col10/1732378460350/Put/seqid=0 2024-11-23T16:14:20,849 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-23T16:14:20,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853. 2024-11-23T16:14:20,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853. as already flushing 2024-11-23T16:14:20,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853. 2024-11-23T16:14:20,849 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => cf5d4648fde06cb520ace381e0be9853, NAME => 'TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:14:20,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => cf5d4648fde06cb520ace381e0be9853, NAME => 'TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:14:20,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cf5d4648fde06cb520ace381e0be9853, NAME => 'TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cf5d4648fde06cb520ace381e0be9853, NAME => 'TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:14:20,854 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:20,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59586 deadline: 1732378520848, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:20,855 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:20,855 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:20,855 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:20,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59638 deadline: 1732378520848, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:20,855 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:14:20,857 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:20,857 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:20,857 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:14:20,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-23T16:14:20,875 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073741986_1162 (size=12151) 2024-11-23T16:14:21,003 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-23T16:14:21,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853. 2024-11-23T16:14:21,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853. as already flushing 2024-11-23T16:14:21,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853. 2024-11-23T16:14:21,003 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => cf5d4648fde06cb520ace381e0be9853, NAME => 'TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:14:21,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => cf5d4648fde06cb520ace381e0be9853, NAME => 'TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:14:21,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cf5d4648fde06cb520ace381e0be9853, NAME => 'TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cf5d4648fde06cb520ace381e0be9853, NAME => 'TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:14:21,092 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:21,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59614 deadline: 1732378521088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:21,092 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:21,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59628 deadline: 1732378521088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:21,093 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:21,093 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:21,093 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:21,093 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:14:21,093 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:21,093 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:14:21,156 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-23T16:14:21,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853. 2024-11-23T16:14:21,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853. as already flushing 2024-11-23T16:14:21,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853. 2024-11-23T16:14:21,160 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => cf5d4648fde06cb520ace381e0be9853, NAME => 'TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:14:21,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => cf5d4648fde06cb520ace381e0be9853, NAME => 'TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:14:21,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cf5d4648fde06cb520ace381e0be9853, NAME => 'TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cf5d4648fde06cb520ace381e0be9853, NAME => 'TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:14:21,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-23T16:14:21,281 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=165 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/.tmp/B/5cfcd5ab3a6249a7ac4da7a37cab896c 2024-11-23T16:14:21,298 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/.tmp/C/09b86ecd3ab346fb95ae235103b74167 is 50, key is test_row_0/C:col10/1732378460350/Put/seqid=0 2024-11-23T16:14:21,315 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-23T16:14:21,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853. 2024-11-23T16:14:21,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853. as already flushing 2024-11-23T16:14:21,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853. 2024-11-23T16:14:21,316 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => cf5d4648fde06cb520ace381e0be9853, NAME => 'TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:14:21,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => cf5d4648fde06cb520ace381e0be9853, NAME => 'TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:14:21,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cf5d4648fde06cb520ace381e0be9853, NAME => 'TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cf5d4648fde06cb520ace381e0be9853, NAME => 'TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:14:21,343 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073741987_1163 (size=12151) 2024-11-23T16:14:21,480 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-23T16:14:21,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853. 2024-11-23T16:14:21,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853. as already flushing 2024-11-23T16:14:21,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853. 2024-11-23T16:14:21,481 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => cf5d4648fde06cb520ace381e0be9853, NAME => 'TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:14:21,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => cf5d4648fde06cb520ace381e0be9853, NAME => 'TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:14:21,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cf5d4648fde06cb520ace381e0be9853, NAME => 'TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cf5d4648fde06cb520ace381e0be9853, NAME => 'TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:14:21,598 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:21,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59628 deadline: 1732378521598, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:21,599 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:21,599 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:21,599 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:14:21,609 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:21,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59614 deadline: 1732378521608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:21,610 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:21,610 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:21,610 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:14:21,634 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-23T16:14:21,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853. 2024-11-23T16:14:21,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853. as already flushing 2024-11-23T16:14:21,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853. 2024-11-23T16:14:21,635 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => cf5d4648fde06cb520ace381e0be9853, NAME => 'TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:14:21,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => cf5d4648fde06cb520ace381e0be9853, NAME => 'TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:14:21,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cf5d4648fde06cb520ace381e0be9853, NAME => 'TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cf5d4648fde06cb520ace381e0be9853, NAME => 'TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:14:21,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-23T16:14:21,743 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=165 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/.tmp/C/09b86ecd3ab346fb95ae235103b74167 2024-11-23T16:14:21,757 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/.tmp/A/274abfa78d934cd49d56bc1cc3ac58fd as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/A/274abfa78d934cd49d56bc1cc3ac58fd 2024-11-23T16:14:21,770 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/A/274abfa78d934cd49d56bc1cc3ac58fd, entries=200, sequenceid=165, filesize=38.8 K 2024-11-23T16:14:21,772 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/.tmp/B/5cfcd5ab3a6249a7ac4da7a37cab896c as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/B/5cfcd5ab3a6249a7ac4da7a37cab896c 2024-11-23T16:14:21,785 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/B/5cfcd5ab3a6249a7ac4da7a37cab896c, entries=150, sequenceid=165, filesize=11.9 K 2024-11-23T16:14:21,786 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/.tmp/C/09b86ecd3ab346fb95ae235103b74167 as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/C/09b86ecd3ab346fb95ae235103b74167 2024-11-23T16:14:21,789 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-23T16:14:21,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853. 2024-11-23T16:14:21,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853. as already flushing 2024-11-23T16:14:21,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853. 2024-11-23T16:14:21,789 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => cf5d4648fde06cb520ace381e0be9853, NAME => 'TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:14:21,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => cf5d4648fde06cb520ace381e0be9853, NAME => 'TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:14:21,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cf5d4648fde06cb520ace381e0be9853, NAME => 'TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cf5d4648fde06cb520ace381e0be9853, NAME => 'TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:14:21,792 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/C/09b86ecd3ab346fb95ae235103b74167, entries=150, sequenceid=165, filesize=11.9 K 2024-11-23T16:14:21,793 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=140.89 KB/144270 for cf5d4648fde06cb520ace381e0be9853 in 1442ms, sequenceid=165, compaction requested=true 2024-11-23T16:14:21,793 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cf5d4648fde06cb520ace381e0be9853: 2024-11-23T16:14:21,793 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cf5d4648fde06cb520ace381e0be9853:A, priority=-2147483648, current under compaction store size is 1 2024-11-23T16:14:21,793 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T16:14:21,793 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T16:14:21,793 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cf5d4648fde06cb520ace381e0be9853:B, priority=-2147483648, current under compaction store size is 2 2024-11-23T16:14:21,794 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T16:14:21,794 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cf5d4648fde06cb520ace381e0be9853:C, priority=-2147483648, current under compaction store size is 3 2024-11-23T16:14:21,794 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T16:14:21,794 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T16:14:21,795 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102156 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T16:14:21,795 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HStore(1541): cf5d4648fde06cb520ace381e0be9853/A is initiating minor compaction (all files) 2024-11-23T16:14:21,795 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cf5d4648fde06cb520ace381e0be9853/A in TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853. 2024-11-23T16:14:21,795 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/A/90db0d62c1304f1f92806b95265a9106, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/A/fb2a54a1f0c640ea8f92799b615c30e9, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/A/274abfa78d934cd49d56bc1cc3ac58fd] into tmpdir=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/.tmp, totalSize=99.8 K 2024-11-23T16:14:21,795 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853. 2024-11-23T16:14:21,795 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853. files: [hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/A/90db0d62c1304f1f92806b95265a9106, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/A/fb2a54a1f0c640ea8f92799b615c30e9, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/A/274abfa78d934cd49d56bc1cc3ac58fd] 2024-11-23T16:14:21,796 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36543 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T16:14:21,796 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HStore(1541): cf5d4648fde06cb520ace381e0be9853/B is initiating minor compaction (all files) 2024-11-23T16:14:21,796 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.Compactor(225): Compacting 90db0d62c1304f1f92806b95265a9106, keycount=150, bloomtype=ROW, size=30.6 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1732378456290 2024-11-23T16:14:21,796 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cf5d4648fde06cb520ace381e0be9853/B in TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853. 2024-11-23T16:14:21,796 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/B/47365aaa2d7e4933b4b0be9429a13d88, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/B/0718a41c649e459a8be955af356e718c, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/B/5cfcd5ab3a6249a7ac4da7a37cab896c] into tmpdir=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/.tmp, totalSize=35.7 K 2024-11-23T16:14:21,796 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.Compactor(225): Compacting fb2a54a1f0c640ea8f92799b615c30e9, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=152, earliestPutTs=1732378458551 2024-11-23T16:14:21,796 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.Compactor(225): Compacting 47365aaa2d7e4933b4b0be9429a13d88, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1732378456290 2024-11-23T16:14:21,797 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.Compactor(225): Compacting 274abfa78d934cd49d56bc1cc3ac58fd, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=165, earliestPutTs=1732378459702 2024-11-23T16:14:21,797 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.Compactor(225): Compacting 0718a41c649e459a8be955af356e718c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=152, earliestPutTs=1732378458551 2024-11-23T16:14:21,798 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.Compactor(225): Compacting 5cfcd5ab3a6249a7ac4da7a37cab896c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=165, earliestPutTs=1732378459706 2024-11-23T16:14:21,826 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cf5d4648fde06cb520ace381e0be9853#B#compaction#138 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T16:14:21,827 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/.tmp/B/34f4da6344f04de99e859f831ed547ea is 50, key is test_row_0/B:col10/1732378460350/Put/seqid=0 2024-11-23T16:14:21,833 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=cf5d4648fde06cb520ace381e0be9853] 2024-11-23T16:14:21,846 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411232da78eeac8d84d77b3e87c19764b4714_cf5d4648fde06cb520ace381e0be9853 store=[table=TestAcidGuarantees family=A region=cf5d4648fde06cb520ace381e0be9853] 2024-11-23T16:14:21,849 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411232da78eeac8d84d77b3e87c19764b4714_cf5d4648fde06cb520ace381e0be9853, store=[table=TestAcidGuarantees family=A region=cf5d4648fde06cb520ace381e0be9853] 2024-11-23T16:14:21,849 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411232da78eeac8d84d77b3e87c19764b4714_cf5d4648fde06cb520ace381e0be9853 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=cf5d4648fde06cb520ace381e0be9853] 2024-11-23T16:14:21,870 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073741988_1164 (size=12493) 2024-11-23T16:14:21,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(8855): Flush requested on cf5d4648fde06cb520ace381e0be9853 2024-11-23T16:14:21,875 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cf5d4648fde06cb520ace381e0be9853 3/3 column families, dataSize=147.60 KB heapSize=406.08 KB 2024-11-23T16:14:21,875 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf5d4648fde06cb520ace381e0be9853, store=A 2024-11-23T16:14:21,875 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-23T16:14:21,875 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf5d4648fde06cb520ace381e0be9853, store=B 2024-11-23T16:14:21,875 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-23T16:14:21,875 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf5d4648fde06cb520ace381e0be9853, store=C 2024-11-23T16:14:21,875 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-23T16:14:21,895 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411233f731c364cfb4ac980412f1b2bfbdf59_cf5d4648fde06cb520ace381e0be9853 is 50, key is test_row_0/A:col10/1732378460387/Put/seqid=0 2024-11-23T16:14:21,898 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:21,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59638 deadline: 1732378521895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:21,899 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:21,899 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:21,899 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:14:21,899 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:21,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59586 deadline: 1732378521897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:21,900 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:21,900 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:21,900 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:14:21,904 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073741990_1166 (size=14794) 2024-11-23T16:14:21,904 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073741989_1165 (size=4469) 2024-11-23T16:14:21,943 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-23T16:14:21,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853. 2024-11-23T16:14:21,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853. as already flushing 2024-11-23T16:14:21,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853. 2024-11-23T16:14:21,943 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => cf5d4648fde06cb520ace381e0be9853, NAME => 'TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:14:21,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => cf5d4648fde06cb520ace381e0be9853, NAME => 'TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:14:21,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cf5d4648fde06cb520ace381e0be9853, NAME => 'TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cf5d4648fde06cb520ace381e0be9853, NAME => 'TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:14:22,010 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:22,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59586 deadline: 1732378522008, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:22,010 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:22,011 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:22,011 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:14:22,010 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:22,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59638 deadline: 1732378522008, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:22,012 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:22,012 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:22,012 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:14:22,097 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-23T16:14:22,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853. 2024-11-23T16:14:22,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853. as already flushing 2024-11-23T16:14:22,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853. 2024-11-23T16:14:22,097 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => cf5d4648fde06cb520ace381e0be9853, NAME => 'TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:14:22,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => cf5d4648fde06cb520ace381e0be9853, NAME => 'TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:14:22,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cf5d4648fde06cb520ace381e0be9853, NAME => 'TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cf5d4648fde06cb520ace381e0be9853, NAME => 'TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:14:22,221 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:22,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59638 deadline: 1732378522218, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:22,222 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:22,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59586 deadline: 1732378522218, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:22,224 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:22,224 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:22,224 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:14:22,224 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:22,224 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:22,224 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:14:22,251 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-23T16:14:22,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853. 2024-11-23T16:14:22,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853. as already flushing 2024-11-23T16:14:22,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853. 2024-11-23T16:14:22,251 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => cf5d4648fde06cb520ace381e0be9853, NAME => 'TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:14:22,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => cf5d4648fde06cb520ace381e0be9853, NAME => 'TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:14:22,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cf5d4648fde06cb520ace381e0be9853, NAME => 'TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cf5d4648fde06cb520ace381e0be9853, NAME => 'TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:14:22,280 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/.tmp/B/34f4da6344f04de99e859f831ed547ea as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/B/34f4da6344f04de99e859f831ed547ea 2024-11-23T16:14:22,287 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cf5d4648fde06cb520ace381e0be9853/B of cf5d4648fde06cb520ace381e0be9853 into 34f4da6344f04de99e859f831ed547ea(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T16:14:22,287 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cf5d4648fde06cb520ace381e0be9853: 2024-11-23T16:14:22,287 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., storeName=cf5d4648fde06cb520ace381e0be9853/B, priority=13, startTime=1732378461793; duration=0sec 2024-11-23T16:14:22,287 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T16:14:22,288 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cf5d4648fde06cb520ace381e0be9853:B 2024-11-23T16:14:22,288 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T16:14:22,289 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36543 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T16:14:22,290 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HStore(1541): cf5d4648fde06cb520ace381e0be9853/C is initiating minor compaction (all files) 2024-11-23T16:14:22,290 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cf5d4648fde06cb520ace381e0be9853/C in TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853. 2024-11-23T16:14:22,290 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/C/cf130f6f9a254c50a6b9c4ebad74e11f, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/C/7ded1e012fcf4760a8d6c12379c832f3, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/C/09b86ecd3ab346fb95ae235103b74167] into tmpdir=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/.tmp, totalSize=35.7 K 2024-11-23T16:14:22,291 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.Compactor(225): Compacting cf130f6f9a254c50a6b9c4ebad74e11f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1732378456290 2024-11-23T16:14:22,291 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.Compactor(225): Compacting 7ded1e012fcf4760a8d6c12379c832f3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=152, earliestPutTs=1732378458551 2024-11-23T16:14:22,292 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.Compactor(225): Compacting 09b86ecd3ab346fb95ae235103b74167, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=165, earliestPutTs=1732378459706 2024-11-23T16:14:22,304 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:22,306 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): cf5d4648fde06cb520ace381e0be9853#A#compaction#139 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T16:14:22,307 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/.tmp/A/0aa01d4258f448abbb96bb919585b2ba is 175, key is test_row_0/A:col10/1732378460350/Put/seqid=0 2024-11-23T16:14:22,312 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cf5d4648fde06cb520ace381e0be9853#C#compaction#141 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T16:14:22,312 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411233f731c364cfb4ac980412f1b2bfbdf59_cf5d4648fde06cb520ace381e0be9853 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411233f731c364cfb4ac980412f1b2bfbdf59_cf5d4648fde06cb520ace381e0be9853 2024-11-23T16:14:22,313 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/.tmp/C/042432554d5e4ca0b7c263ac62dcdccc is 50, key is test_row_0/C:col10/1732378460350/Put/seqid=0 2024-11-23T16:14:22,314 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/.tmp/A/040a9bb41af84f1dbe6a0145ef6dd3d2, store: [table=TestAcidGuarantees family=A region=cf5d4648fde06cb520ace381e0be9853] 2024-11-23T16:14:22,314 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/.tmp/A/040a9bb41af84f1dbe6a0145ef6dd3d2 is 175, key is test_row_0/A:col10/1732378460387/Put/seqid=0 2024-11-23T16:14:22,335 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073741991_1167 (size=31447) 2024-11-23T16:14:22,338 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073741992_1168 (size=39749) 2024-11-23T16:14:22,339 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=200, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/.tmp/A/040a9bb41af84f1dbe6a0145ef6dd3d2 2024-11-23T16:14:22,354 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073741993_1169 (size=12493) 2024-11-23T16:14:22,363 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/.tmp/C/042432554d5e4ca0b7c263ac62dcdccc as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/C/042432554d5e4ca0b7c263ac62dcdccc 2024-11-23T16:14:22,371 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cf5d4648fde06cb520ace381e0be9853/C of cf5d4648fde06cb520ace381e0be9853 into 042432554d5e4ca0b7c263ac62dcdccc(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T16:14:22,371 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cf5d4648fde06cb520ace381e0be9853: 2024-11-23T16:14:22,371 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., storeName=cf5d4648fde06cb520ace381e0be9853/C, priority=13, startTime=1732378461794; duration=0sec 2024-11-23T16:14:22,371 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T16:14:22,371 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cf5d4648fde06cb520ace381e0be9853:C 2024-11-23T16:14:22,379 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/.tmp/B/62abb0de1c19491e98b7b8869e639b0c is 50, key is test_row_0/B:col10/1732378460387/Put/seqid=0 2024-11-23T16:14:22,405 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-23T16:14:22,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853. 2024-11-23T16:14:22,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853. as already flushing 2024-11-23T16:14:22,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853. 2024-11-23T16:14:22,406 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => cf5d4648fde06cb520ace381e0be9853, NAME => 'TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:14:22,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => cf5d4648fde06cb520ace381e0be9853, NAME => 'TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:14:22,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cf5d4648fde06cb520ace381e0be9853, NAME => 'TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cf5d4648fde06cb520ace381e0be9853, NAME => 'TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:14:22,414 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073741994_1170 (size=12151) 2024-11-23T16:14:22,415 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=200 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/.tmp/B/62abb0de1c19491e98b7b8869e639b0c 2024-11-23T16:14:22,426 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/.tmp/C/3ee5535d7f49438f9e13c9f43178ffb3 is 50, key is test_row_0/C:col10/1732378460387/Put/seqid=0 2024-11-23T16:14:22,461 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073741995_1171 (size=12151) 2024-11-23T16:14:22,463 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=200 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/.tmp/C/3ee5535d7f49438f9e13c9f43178ffb3 2024-11-23T16:14:22,470 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/.tmp/A/040a9bb41af84f1dbe6a0145ef6dd3d2 as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/A/040a9bb41af84f1dbe6a0145ef6dd3d2 2024-11-23T16:14:22,475 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/A/040a9bb41af84f1dbe6a0145ef6dd3d2, entries=200, sequenceid=200, filesize=38.8 K 2024-11-23T16:14:22,476 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/.tmp/B/62abb0de1c19491e98b7b8869e639b0c as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/B/62abb0de1c19491e98b7b8869e639b0c 2024-11-23T16:14:22,482 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/B/62abb0de1c19491e98b7b8869e639b0c, entries=150, sequenceid=200, filesize=11.9 K 2024-11-23T16:14:22,483 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/.tmp/C/3ee5535d7f49438f9e13c9f43178ffb3 as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/C/3ee5535d7f49438f9e13c9f43178ffb3 2024-11-23T16:14:22,488 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/C/3ee5535d7f49438f9e13c9f43178ffb3, entries=150, sequenceid=200, filesize=11.9 K 2024-11-23T16:14:22,489 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~406.03 KB/415776, currentSize=40.25 KB/41220 for cf5d4648fde06cb520ace381e0be9853 in 614ms, sequenceid=200, compaction requested=false 2024-11-23T16:14:22,489 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cf5d4648fde06cb520ace381e0be9853: 2024-11-23T16:14:22,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(8855): Flush requested on cf5d4648fde06cb520ace381e0be9853 2024-11-23T16:14:22,535 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cf5d4648fde06cb520ace381e0be9853 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-23T16:14:22,536 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf5d4648fde06cb520ace381e0be9853, store=A 2024-11-23T16:14:22,536 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-23T16:14:22,536 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf5d4648fde06cb520ace381e0be9853, store=B 2024-11-23T16:14:22,536 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-23T16:14:22,536 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf5d4648fde06cb520ace381e0be9853, store=C 2024-11-23T16:14:22,536 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-23T16:14:22,551 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241123b592ad9a15284f189537a0f832052560_cf5d4648fde06cb520ace381e0be9853 is 50, key is test_row_0/A:col10/1732378462532/Put/seqid=0 2024-11-23T16:14:22,565 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-23T16:14:22,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853. 2024-11-23T16:14:22,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853. as already flushing 2024-11-23T16:14:22,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853. 2024-11-23T16:14:22,566 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => cf5d4648fde06cb520ace381e0be9853, NAME => 'TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:14:22,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => cf5d4648fde06cb520ace381e0be9853, NAME => 'TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:14:22,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cf5d4648fde06cb520ace381e0be9853, NAME => 'TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cf5d4648fde06cb520ace381e0be9853, NAME => 'TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:14:22,591 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073741996_1172 (size=14794) 2024-11-23T16:14:22,609 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1a7c5470b472:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732378462590, max timestamp=1732378462605Num uniques 859; 2024-11-23T16:14:22,609 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1a7c5470b472:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732378462590, max timestamp=1732378462605Num uniques 859; 2024-11-23T16:14:22,609 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1a7c5470b472:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-23T16:14:22,609 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1a7c5470b472:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-23T16:14:22,609 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1a7c5470b472:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732378462590, max timestamp=1732378462605Num uniques 859; 2024-11-23T16:14:22,609 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1a7c5470b472:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-23T16:14:22,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-23T16:14:22,703 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:22,703 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:22,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59638 deadline: 1732378522688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:22,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59614 deadline: 1732378522689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:22,703 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:22,703 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:22,704 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:14:22,703 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:22,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59628 deadline: 1732378522690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:22,704 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:22,704 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:22,704 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:14:22,704 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:22,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59586 deadline: 1732378522698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:22,705 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:22,705 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:22,705 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:14:22,705 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:22,705 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:22,705 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:14:22,718 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-23T16:14:22,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853. 2024-11-23T16:14:22,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853. as already flushing 2024-11-23T16:14:22,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853. 2024-11-23T16:14:22,719 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => cf5d4648fde06cb520ace381e0be9853, NAME => 'TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:14:22,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => cf5d4648fde06cb520ace381e0be9853, NAME => 'TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:14:22,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cf5d4648fde06cb520ace381e0be9853, NAME => 'TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cf5d4648fde06cb520ace381e0be9853, NAME => 'TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:14:22,749 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/.tmp/A/0aa01d4258f448abbb96bb919585b2ba as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/A/0aa01d4258f448abbb96bb919585b2ba 2024-11-23T16:14:22,752 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:22,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59584 deadline: 1732378522748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:22,753 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to 1a7c5470b472,44715,1732378420375 for 'test_row_1' in cf5d4648fde06cb520ace381e0be9853 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4189 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-23T16:14:22,754 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:22,754 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:22,754 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:14:22,759 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cf5d4648fde06cb520ace381e0be9853/A of cf5d4648fde06cb520ace381e0be9853 into 0aa01d4258f448abbb96bb919585b2ba(size=30.7 K), total size for store is 69.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T16:14:22,759 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cf5d4648fde06cb520ace381e0be9853: 2024-11-23T16:14:22,759 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., storeName=cf5d4648fde06cb520ace381e0be9853/A, priority=13, startTime=1732378461793; duration=0sec 2024-11-23T16:14:22,759 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T16:14:22,759 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cf5d4648fde06cb520ace381e0be9853:A 2024-11-23T16:14:22,815 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:22,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59638 deadline: 1732378522808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:22,816 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:22,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59628 deadline: 1732378522809, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:22,816 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:22,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59586 deadline: 1732378522809, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:22,817 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:22,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59614 deadline: 1732378522809, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:22,818 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:22,818 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:22,818 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:14:22,818 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:22,818 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:22,818 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:14:22,819 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:22,819 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:22,819 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:14:22,819 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:22,819 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:22,819 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:14:22,877 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-23T16:14:22,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853. 2024-11-23T16:14:22,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853. as already flushing 2024-11-23T16:14:22,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853. 2024-11-23T16:14:22,878 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => cf5d4648fde06cb520ace381e0be9853, NAME => 'TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:14:22,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => cf5d4648fde06cb520ace381e0be9853, NAME => 'TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:14:22,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cf5d4648fde06cb520ace381e0be9853, NAME => 'TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cf5d4648fde06cb520ace381e0be9853, NAME => 'TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:14:22,991 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:23,017 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241123b592ad9a15284f189537a0f832052560_cf5d4648fde06cb520ace381e0be9853 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123b592ad9a15284f189537a0f832052560_cf5d4648fde06cb520ace381e0be9853 2024-11-23T16:14:23,019 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/.tmp/A/fbde40db105a4a57a29f1abe87a38595, store: [table=TestAcidGuarantees family=A region=cf5d4648fde06cb520ace381e0be9853] 2024-11-23T16:14:23,020 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/.tmp/A/fbde40db105a4a57a29f1abe87a38595 is 175, key is test_row_0/A:col10/1732378462532/Put/seqid=0 2024-11-23T16:14:23,025 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073741997_1173 (size=39749) 2024-11-23T16:14:23,026 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=213, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/.tmp/A/fbde40db105a4a57a29f1abe87a38595 2024-11-23T16:14:23,031 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-23T16:14:23,031 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:23,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853. 2024-11-23T16:14:23,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59586 deadline: 1732378523028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:23,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853. as already flushing 2024-11-23T16:14:23,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853. 2024-11-23T16:14:23,032 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => cf5d4648fde06cb520ace381e0be9853, NAME => 'TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:14:23,032 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:23,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => cf5d4648fde06cb520ace381e0be9853, NAME => 'TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:14:23,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59628 deadline: 1732378523028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:23,032 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:23,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59614 deadline: 1732378523028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:23,032 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:23,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59638 deadline: 1732378523028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:23,033 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:23,033 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:23,033 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:14:23,033 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:23,033 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:23,033 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:14:23,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cf5d4648fde06cb520ace381e0be9853, NAME => 'TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cf5d4648fde06cb520ace381e0be9853, NAME => 'TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:14:23,034 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:23,034 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:23,034 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:14:23,034 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:23,034 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:23,034 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:14:23,040 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/.tmp/B/c6d367246e934bdaa61003a704765ac1 is 50, key is test_row_0/B:col10/1732378462532/Put/seqid=0 2024-11-23T16:14:23,050 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073741998_1174 (size=12151) 2024-11-23T16:14:23,051 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=213 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/.tmp/B/c6d367246e934bdaa61003a704765ac1 2024-11-23T16:14:23,061 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/.tmp/C/09458dc1927d49249fb9ce5f661bb94d is 50, key is test_row_0/C:col10/1732378462532/Put/seqid=0 2024-11-23T16:14:23,066 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073741999_1175 (size=12151) 2024-11-23T16:14:23,186 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-23T16:14:23,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853. 2024-11-23T16:14:23,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853. as already flushing 2024-11-23T16:14:23,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853. 2024-11-23T16:14:23,187 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => cf5d4648fde06cb520ace381e0be9853, NAME => 'TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:14:23,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => cf5d4648fde06cb520ace381e0be9853, NAME => 'TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:14:23,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cf5d4648fde06cb520ace381e0be9853, NAME => 'TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cf5d4648fde06cb520ace381e0be9853, NAME => 'TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:14:23,340 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-23T16:14:23,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853. 2024-11-23T16:14:23,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853. as already flushing 2024-11-23T16:14:23,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853. 2024-11-23T16:14:23,340 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => cf5d4648fde06cb520ace381e0be9853, NAME => 'TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:14:23,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => cf5d4648fde06cb520ace381e0be9853, NAME => 'TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:14:23,341 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:23,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59638 deadline: 1732378523338, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:23,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cf5d4648fde06cb520ace381e0be9853, NAME => 'TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cf5d4648fde06cb520ace381e0be9853, NAME => 'TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:14:23,342 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:23,342 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:23,342 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:14:23,342 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:23,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59614 deadline: 1732378523339, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:23,343 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:23,343 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:23,343 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:23,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59628 deadline: 1732378523339, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:23,343 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:14:23,343 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:23,344 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:23,344 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:23,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59586 deadline: 1732378523339, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:23,344 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:14:23,344 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:23,344 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:23,344 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:14:23,468 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=213 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/.tmp/C/09458dc1927d49249fb9ce5f661bb94d 2024-11-23T16:14:23,473 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/.tmp/A/fbde40db105a4a57a29f1abe87a38595 as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/A/fbde40db105a4a57a29f1abe87a38595 2024-11-23T16:14:23,478 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/A/fbde40db105a4a57a29f1abe87a38595, entries=200, sequenceid=213, filesize=38.8 K 2024-11-23T16:14:23,480 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/.tmp/B/c6d367246e934bdaa61003a704765ac1 as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/B/c6d367246e934bdaa61003a704765ac1 2024-11-23T16:14:23,485 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/B/c6d367246e934bdaa61003a704765ac1, entries=150, sequenceid=213, filesize=11.9 K 2024-11-23T16:14:23,487 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/.tmp/C/09458dc1927d49249fb9ce5f661bb94d as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/C/09458dc1927d49249fb9ce5f661bb94d 2024-11-23T16:14:23,492 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/C/09458dc1927d49249fb9ce5f661bb94d, entries=150, sequenceid=213, filesize=11.9 K 2024-11-23T16:14:23,493 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for cf5d4648fde06cb520ace381e0be9853 in 958ms, sequenceid=213, compaction requested=true 2024-11-23T16:14:23,493 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cf5d4648fde06cb520ace381e0be9853: 2024-11-23T16:14:23,493 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cf5d4648fde06cb520ace381e0be9853:A, priority=-2147483648, current under compaction store size is 1 2024-11-23T16:14:23,493 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T16:14:23,493 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cf5d4648fde06cb520ace381e0be9853:B, priority=-2147483648, current under compaction store size is 2 2024-11-23T16:14:23,493 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T16:14:23,493 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T16:14:23,493 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cf5d4648fde06cb520ace381e0be9853:C, priority=-2147483648, current under compaction store size is 3 2024-11-23T16:14:23,493 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-23T16:14:23,493 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-23T16:14:23,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853. 2024-11-23T16:14:23,494 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T16:14:23,494 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2902): Flushing cf5d4648fde06cb520ace381e0be9853 3/3 column families, dataSize=127.47 KB heapSize=360.38 KB 2024-11-23T16:14:23,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf5d4648fde06cb520ace381e0be9853, store=A 2024-11-23T16:14:23,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-23T16:14:23,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf5d4648fde06cb520ace381e0be9853, store=B 2024-11-23T16:14:23,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-23T16:14:23,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf5d4648fde06cb520ace381e0be9853, store=C 2024-11-23T16:14:23,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-23T16:14:23,495 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110945 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T16:14:23,495 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HStore(1541): cf5d4648fde06cb520ace381e0be9853/A is initiating minor compaction (all files) 2024-11-23T16:14:23,495 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cf5d4648fde06cb520ace381e0be9853/A in TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853. 2024-11-23T16:14:23,495 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/A/0aa01d4258f448abbb96bb919585b2ba, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/A/040a9bb41af84f1dbe6a0145ef6dd3d2, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/A/fbde40db105a4a57a29f1abe87a38595] into tmpdir=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/.tmp, totalSize=108.3 K 2024-11-23T16:14:23,495 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853. 2024-11-23T16:14:23,495 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853. files: [hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/A/0aa01d4258f448abbb96bb919585b2ba, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/A/040a9bb41af84f1dbe6a0145ef6dd3d2, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/A/fbde40db105a4a57a29f1abe87a38595] 2024-11-23T16:14:23,496 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T16:14:23,496 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HStore(1541): cf5d4648fde06cb520ace381e0be9853/B is initiating minor compaction (all files) 2024-11-23T16:14:23,496 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cf5d4648fde06cb520ace381e0be9853/B in TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853. 2024-11-23T16:14:23,496 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/B/34f4da6344f04de99e859f831ed547ea, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/B/62abb0de1c19491e98b7b8869e639b0c, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/B/c6d367246e934bdaa61003a704765ac1] into tmpdir=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/.tmp, totalSize=35.9 K 2024-11-23T16:14:23,497 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.Compactor(225): Compacting 0aa01d4258f448abbb96bb919585b2ba, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=165, earliestPutTs=1732378459706 2024-11-23T16:14:23,497 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.Compactor(225): Compacting 34f4da6344f04de99e859f831ed547ea, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=165, earliestPutTs=1732378459706 2024-11-23T16:14:23,497 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.Compactor(225): Compacting 040a9bb41af84f1dbe6a0145ef6dd3d2, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=200, earliestPutTs=1732378460387 2024-11-23T16:14:23,499 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.Compactor(225): Compacting 62abb0de1c19491e98b7b8869e639b0c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=200, earliestPutTs=1732378460387 2024-11-23T16:14:23,499 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.Compactor(225): Compacting fbde40db105a4a57a29f1abe87a38595, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1732378461888 2024-11-23T16:14:23,501 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.Compactor(225): Compacting c6d367246e934bdaa61003a704765ac1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1732378461895 2024-11-23T16:14:23,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241123f0ab74254bad4613981a072935329952_cf5d4648fde06cb520ace381e0be9853 is 50, key is test_row_0/A:col10/1732378462686/Put/seqid=0 2024-11-23T16:14:23,523 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): cf5d4648fde06cb520ace381e0be9853#B#compaction#148 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T16:14:23,523 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/.tmp/B/d6a4fa3d2a37434b9de8157cc23f73bf is 50, key is test_row_0/B:col10/1732378462532/Put/seqid=0 2024-11-23T16:14:23,533 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=cf5d4648fde06cb520ace381e0be9853] 2024-11-23T16:14:23,544 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411233273fa0e6c7a4079a93cf1a855945b08_cf5d4648fde06cb520ace381e0be9853 store=[table=TestAcidGuarantees family=A region=cf5d4648fde06cb520ace381e0be9853] 2024-11-23T16:14:23,546 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411233273fa0e6c7a4079a93cf1a855945b08_cf5d4648fde06cb520ace381e0be9853, store=[table=TestAcidGuarantees family=A region=cf5d4648fde06cb520ace381e0be9853] 2024-11-23T16:14:23,547 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411233273fa0e6c7a4079a93cf1a855945b08_cf5d4648fde06cb520ace381e0be9853 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=cf5d4648fde06cb520ace381e0be9853] 2024-11-23T16:14:23,555 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742001_1177 (size=12595) 2024-11-23T16:14:23,569 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/.tmp/B/d6a4fa3d2a37434b9de8157cc23f73bf as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/B/d6a4fa3d2a37434b9de8157cc23f73bf 2024-11-23T16:14:23,584 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742000_1176 (size=12304) 2024-11-23T16:14:23,587 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cf5d4648fde06cb520ace381e0be9853/B of cf5d4648fde06cb520ace381e0be9853 into d6a4fa3d2a37434b9de8157cc23f73bf(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T16:14:23,587 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cf5d4648fde06cb520ace381e0be9853: 2024-11-23T16:14:23,587 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., storeName=cf5d4648fde06cb520ace381e0be9853/B, priority=13, startTime=1732378463493; duration=0sec 2024-11-23T16:14:23,587 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T16:14:23,587 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cf5d4648fde06cb520ace381e0be9853:B 2024-11-23T16:14:23,587 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T16:14:23,588 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T16:14:23,588 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HStore(1541): cf5d4648fde06cb520ace381e0be9853/C is initiating minor compaction (all files) 2024-11-23T16:14:23,588 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cf5d4648fde06cb520ace381e0be9853/C in TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853. 2024-11-23T16:14:23,589 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/C/042432554d5e4ca0b7c263ac62dcdccc, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/C/3ee5535d7f49438f9e13c9f43178ffb3, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/C/09458dc1927d49249fb9ce5f661bb94d] into tmpdir=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/.tmp, totalSize=35.9 K 2024-11-23T16:14:23,590 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742002_1178 (size=4469) 2024-11-23T16:14:23,590 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.Compactor(225): Compacting 042432554d5e4ca0b7c263ac62dcdccc, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=165, earliestPutTs=1732378459706 2024-11-23T16:14:23,592 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3ee5535d7f49438f9e13c9f43178ffb3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=200, earliestPutTs=1732378460387 2024-11-23T16:14:23,593 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cf5d4648fde06cb520ace381e0be9853#A#compaction#149 average throughput is 0.41 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T16:14:23,593 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/.tmp/A/63aca72b8ed64727b1f106e1ae4552b3 is 175, key is test_row_0/A:col10/1732378462532/Put/seqid=0 2024-11-23T16:14:23,594 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.Compactor(225): Compacting 09458dc1927d49249fb9ce5f661bb94d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1732378461895 2024-11-23T16:14:23,604 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): cf5d4648fde06cb520ace381e0be9853#C#compaction#150 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T16:14:23,605 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/.tmp/C/deec6a92bb0f4215b405bcfd187e95c9 is 50, key is test_row_0/C:col10/1732378462532/Put/seqid=0 2024-11-23T16:14:23,615 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742003_1179 (size=31549) 2024-11-23T16:14:23,622 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742004_1180 (size=12595) 2024-11-23T16:14:23,629 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/.tmp/C/deec6a92bb0f4215b405bcfd187e95c9 as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/C/deec6a92bb0f4215b405bcfd187e95c9 2024-11-23T16:14:23,635 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cf5d4648fde06cb520ace381e0be9853/C of cf5d4648fde06cb520ace381e0be9853 into deec6a92bb0f4215b405bcfd187e95c9(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T16:14:23,635 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cf5d4648fde06cb520ace381e0be9853: 2024-11-23T16:14:23,635 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., storeName=cf5d4648fde06cb520ace381e0be9853/C, priority=13, startTime=1732378463493; duration=0sec 2024-11-23T16:14:23,635 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T16:14:23,636 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cf5d4648fde06cb520ace381e0be9853:C 2024-11-23T16:14:23,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(8855): Flush requested on cf5d4648fde06cb520ace381e0be9853 2024-11-23T16:14:23,850 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853. as already flushing 2024-11-23T16:14:23,862 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:23,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59614 deadline: 1732378523858, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:23,863 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:23,863 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:23,863 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:23,863 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:14:23,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59586 deadline: 1732378523860, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:23,863 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:23,863 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:23,863 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:14:23,866 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:23,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59638 deadline: 1732378523861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:23,866 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:23,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59628 deadline: 1732378523862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:23,867 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:23,867 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:23,867 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:14:23,868 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:23,868 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:23,868 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:14:23,948 INFO [master/1a7c5470b472:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-11-23T16:14:23,948 INFO [master/1a7c5470b472:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-11-23T16:14:23,973 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:23,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59586 deadline: 1732378523968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:23,973 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:23,973 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:23,973 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:14:23,984 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:23,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59628 deadline: 1732378523978, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:23,985 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:23,985 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:23,985 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:14:23,985 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:23,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59638 deadline: 1732378523978, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:23,985 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:23,985 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:23,985 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:14:23,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:23,996 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241123f0ab74254bad4613981a072935329952_cf5d4648fde06cb520ace381e0be9853 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123f0ab74254bad4613981a072935329952_cf5d4648fde06cb520ace381e0be9853 2024-11-23T16:14:23,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/.tmp/A/5477215922a748ef8f1ee5c942bf9e3f, store: [table=TestAcidGuarantees family=A region=cf5d4648fde06cb520ace381e0be9853] 2024-11-23T16:14:23,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/.tmp/A/5477215922a748ef8f1ee5c942bf9e3f is 175, key is test_row_0/A:col10/1732378462686/Put/seqid=0 2024-11-23T16:14:24,006 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742005_1181 (size=31105) 2024-11-23T16:14:24,022 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/.tmp/A/63aca72b8ed64727b1f106e1ae4552b3 as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/A/63aca72b8ed64727b1f106e1ae4552b3 2024-11-23T16:14:24,029 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cf5d4648fde06cb520ace381e0be9853/A of cf5d4648fde06cb520ace381e0be9853 into 63aca72b8ed64727b1f106e1ae4552b3(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T16:14:24,029 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cf5d4648fde06cb520ace381e0be9853: 2024-11-23T16:14:24,029 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., storeName=cf5d4648fde06cb520ace381e0be9853/A, priority=13, startTime=1732378463493; duration=0sec 2024-11-23T16:14:24,029 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T16:14:24,029 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cf5d4648fde06cb520ace381e0be9853:A 2024-11-23T16:14:24,186 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:24,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59586 deadline: 1732378524178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:24,187 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:24,187 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:24,187 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:14:24,189 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:24,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59628 deadline: 1732378524188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:24,189 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:24,189 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:24,189 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:14:24,203 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:24,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59638 deadline: 1732378524197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:24,203 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:24,203 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:24,203 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:14:24,407 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=249, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/.tmp/A/5477215922a748ef8f1ee5c942bf9e3f 2024-11-23T16:14:24,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/.tmp/B/787bc5f7e6e34bf7b0e06e9da46894c1 is 50, key is test_row_0/B:col10/1732378462686/Put/seqid=0 2024-11-23T16:14:24,426 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742006_1182 (size=12151) 2024-11-23T16:14:24,437 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=249 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/.tmp/B/787bc5f7e6e34bf7b0e06e9da46894c1 2024-11-23T16:14:24,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/.tmp/C/4279c0d888624129b950c7373b30fca9 is 50, key is test_row_0/C:col10/1732378462686/Put/seqid=0 2024-11-23T16:14:24,460 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742007_1183 (size=12151) 2024-11-23T16:14:24,461 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=249 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/.tmp/C/4279c0d888624129b950c7373b30fca9 2024-11-23T16:14:24,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/.tmp/A/5477215922a748ef8f1ee5c942bf9e3f as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/A/5477215922a748ef8f1ee5c942bf9e3f 2024-11-23T16:14:24,477 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/A/5477215922a748ef8f1ee5c942bf9e3f, entries=150, sequenceid=249, filesize=30.4 K 2024-11-23T16:14:24,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/.tmp/B/787bc5f7e6e34bf7b0e06e9da46894c1 as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/B/787bc5f7e6e34bf7b0e06e9da46894c1 2024-11-23T16:14:24,482 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/B/787bc5f7e6e34bf7b0e06e9da46894c1, entries=150, sequenceid=249, filesize=11.9 K 2024-11-23T16:14:24,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/.tmp/C/4279c0d888624129b950c7373b30fca9 as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/C/4279c0d888624129b950c7373b30fca9 2024-11-23T16:14:24,498 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/C/4279c0d888624129b950c7373b30fca9, entries=150, sequenceid=249, filesize=11.9 K 2024-11-23T16:14:24,499 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~360.33 KB/368976, currentSize=53.67 KB/54960 for cf5d4648fde06cb520ace381e0be9853 in 1005ms, sequenceid=249, compaction requested=false 2024-11-23T16:14:24,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2603): Flush status journal for cf5d4648fde06cb520ace381e0be9853: 2024-11-23T16:14:24,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853. 2024-11-23T16:14:24,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=47 2024-11-23T16:14:24,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.HMaster(4169): Remote procedure done, pid=47 2024-11-23T16:14:24,507 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=47, resume processing ppid=46 2024-11-23T16:14:24,507 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, ppid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.9630 sec 2024-11-23T16:14:24,509 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees in 3.9690 sec 2024-11-23T16:14:24,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(8855): Flush requested on cf5d4648fde06cb520ace381e0be9853 2024-11-23T16:14:24,513 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cf5d4648fde06cb520ace381e0be9853 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-23T16:14:24,513 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf5d4648fde06cb520ace381e0be9853, store=A 2024-11-23T16:14:24,513 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-23T16:14:24,513 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf5d4648fde06cb520ace381e0be9853, store=B 2024-11-23T16:14:24,513 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-23T16:14:24,513 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf5d4648fde06cb520ace381e0be9853, store=C 2024-11-23T16:14:24,513 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-23T16:14:24,538 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241123e548afcfc9214176ad55568064585f18_cf5d4648fde06cb520ace381e0be9853 is 50, key is test_row_0/A:col10/1732378464503/Put/seqid=0 2024-11-23T16:14:24,553 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:24,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59628 deadline: 1732378524548, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:24,553 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:24,553 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:24,554 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:14:24,560 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:24,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59638 deadline: 1732378524552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:24,560 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:24,560 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:24,560 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:14:24,561 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:24,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59586 deadline: 1732378524553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:24,561 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:24,561 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:24,561 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:14:24,562 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742008_1184 (size=12404) 2024-11-23T16:14:24,564 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:24,577 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241123e548afcfc9214176ad55568064585f18_cf5d4648fde06cb520ace381e0be9853 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123e548afcfc9214176ad55568064585f18_cf5d4648fde06cb520ace381e0be9853 2024-11-23T16:14:24,578 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/.tmp/A/547aea4f4f6e4109818f11a11d224b63, store: [table=TestAcidGuarantees family=A region=cf5d4648fde06cb520ace381e0be9853] 2024-11-23T16:14:24,579 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/.tmp/A/547aea4f4f6e4109818f11a11d224b63 is 175, key is test_row_0/A:col10/1732378464503/Put/seqid=0 2024-11-23T16:14:24,590 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742009_1185 (size=31205) 2024-11-23T16:14:24,591 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=265, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/.tmp/A/547aea4f4f6e4109818f11a11d224b63 2024-11-23T16:14:24,607 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/.tmp/B/e2a75731746a43719653ec1be70a2a2b is 50, key is test_row_0/B:col10/1732378464503/Put/seqid=0 2024-11-23T16:14:24,612 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742010_1186 (size=12251) 2024-11-23T16:14:24,658 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:24,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59628 deadline: 1732378524658, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:24,659 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:24,659 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:24,659 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:14:24,672 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:24,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59638 deadline: 1732378524668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:24,672 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:24,672 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:24,672 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:24,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59586 deadline: 1732378524668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:24,672 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:14:24,673 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:24,673 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:24,673 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:14:24,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-23T16:14:24,709 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-23T16:14:24,709 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-23T16:14:24,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] procedure2.ProcedureExecutor(1139): Stored pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees 2024-11-23T16:14:24,712 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-23T16:14:24,712 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-23T16:14:24,713 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=49, ppid=48, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-23T16:14:24,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-23T16:14:24,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-23T16:14:24,868 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-23T16:14:24,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853. 2024-11-23T16:14:24,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853. as already flushing 2024-11-23T16:14:24,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853. 2024-11-23T16:14:24,869 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => cf5d4648fde06cb520ace381e0be9853, NAME => 'TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:14:24,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => cf5d4648fde06cb520ace381e0be9853, NAME => 'TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:14:24,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cf5d4648fde06cb520ace381e0be9853, NAME => 'TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cf5d4648fde06cb520ace381e0be9853, NAME => 'TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:14:24,872 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:24,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59628 deadline: 1732378524868, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:24,873 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:24,873 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:24,873 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:14:24,881 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:24,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59614 deadline: 1732378524878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:24,881 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:24,881 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:24,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59638 deadline: 1732378524878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:24,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59586 deadline: 1732378524878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:24,881 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 1a7c5470b472,44715,1732378420375 for 'test_row_1' in cf5d4648fde06cb520ace381e0be9853 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2192 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-23T16:14:24,882 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:24,882 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:24,882 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:14:24,882 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:24,882 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:24,882 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:14:24,882 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:24,882 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:24,883 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:14:25,013 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=265 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/.tmp/B/e2a75731746a43719653ec1be70a2a2b 2024-11-23T16:14:25,022 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-23T16:14:25,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853. 2024-11-23T16:14:25,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853. as already flushing 2024-11-23T16:14:25,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853. 2024-11-23T16:14:25,022 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => cf5d4648fde06cb520ace381e0be9853, NAME => 'TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:14:25,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => cf5d4648fde06cb520ace381e0be9853, NAME => 'TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:14:25,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cf5d4648fde06cb520ace381e0be9853, NAME => 'TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cf5d4648fde06cb520ace381e0be9853, NAME => 'TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:14:25,030 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/.tmp/C/db8361a1cb654eb9aa52e87c1563755b is 50, key is test_row_0/C:col10/1732378464503/Put/seqid=0 2024-11-23T16:14:25,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-23T16:14:25,038 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742011_1187 (size=12251) 2024-11-23T16:14:25,176 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-23T16:14:25,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853. 2024-11-23T16:14:25,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853. as already flushing 2024-11-23T16:14:25,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853. 2024-11-23T16:14:25,180 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => cf5d4648fde06cb520ace381e0be9853, NAME => 'TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:14:25,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => cf5d4648fde06cb520ace381e0be9853, NAME => 'TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:14:25,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cf5d4648fde06cb520ace381e0be9853, NAME => 'TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cf5d4648fde06cb520ace381e0be9853, NAME => 'TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:14:25,184 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:25,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59628 deadline: 1732378525181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:25,185 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:25,185 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:25,185 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:14:25,192 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:25,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59586 deadline: 1732378525188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:25,192 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:25,192 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:25,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59638 deadline: 1732378525188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:25,193 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:25,193 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:14:25,193 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:25,193 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:25,193 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:14:25,335 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-23T16:14:25,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853. 2024-11-23T16:14:25,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853. as already flushing 2024-11-23T16:14:25,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853. 2024-11-23T16:14:25,335 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => cf5d4648fde06cb520ace381e0be9853, NAME => 'TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:14:25,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => cf5d4648fde06cb520ace381e0be9853, NAME => 'TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:14:25,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cf5d4648fde06cb520ace381e0be9853, NAME => 'TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cf5d4648fde06cb520ace381e0be9853, NAME => 'TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:14:25,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-23T16:14:25,439 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=265 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/.tmp/C/db8361a1cb654eb9aa52e87c1563755b 2024-11-23T16:14:25,464 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/.tmp/A/547aea4f4f6e4109818f11a11d224b63 as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/A/547aea4f4f6e4109818f11a11d224b63 2024-11-23T16:14:25,471 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/A/547aea4f4f6e4109818f11a11d224b63, entries=150, sequenceid=265, filesize=30.5 K 2024-11-23T16:14:25,473 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/.tmp/B/e2a75731746a43719653ec1be70a2a2b as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/B/e2a75731746a43719653ec1be70a2a2b 2024-11-23T16:14:25,481 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/B/e2a75731746a43719653ec1be70a2a2b, entries=150, sequenceid=265, filesize=12.0 K 2024-11-23T16:14:25,482 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/.tmp/C/db8361a1cb654eb9aa52e87c1563755b as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/C/db8361a1cb654eb9aa52e87c1563755b 2024-11-23T16:14:25,488 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-23T16:14:25,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853. 2024-11-23T16:14:25,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853. as already flushing 2024-11-23T16:14:25,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853. 2024-11-23T16:14:25,488 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => cf5d4648fde06cb520ace381e0be9853, NAME => 'TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:14:25,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => cf5d4648fde06cb520ace381e0be9853, NAME => 'TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:14:25,489 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/C/db8361a1cb654eb9aa52e87c1563755b, entries=150, sequenceid=265, filesize=12.0 K 2024-11-23T16:14:25,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cf5d4648fde06cb520ace381e0be9853, NAME => 'TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cf5d4648fde06cb520ace381e0be9853, NAME => 'TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:14:25,490 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for cf5d4648fde06cb520ace381e0be9853 in 977ms, sequenceid=265, compaction requested=true 2024-11-23T16:14:25,491 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cf5d4648fde06cb520ace381e0be9853: 2024-11-23T16:14:25,491 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T16:14:25,492 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93859 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T16:14:25,492 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HStore(1541): cf5d4648fde06cb520ace381e0be9853/A is initiating minor compaction (all files) 2024-11-23T16:14:25,492 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cf5d4648fde06cb520ace381e0be9853/A in TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853. 2024-11-23T16:14:25,492 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/A/63aca72b8ed64727b1f106e1ae4552b3, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/A/5477215922a748ef8f1ee5c942bf9e3f, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/A/547aea4f4f6e4109818f11a11d224b63] into tmpdir=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/.tmp, totalSize=91.7 K 2024-11-23T16:14:25,492 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853. 2024-11-23T16:14:25,493 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853. files: [hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/A/63aca72b8ed64727b1f106e1ae4552b3, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/A/5477215922a748ef8f1ee5c942bf9e3f, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/A/547aea4f4f6e4109818f11a11d224b63] 2024-11-23T16:14:25,493 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.Compactor(225): Compacting 63aca72b8ed64727b1f106e1ae4552b3, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1732378461895 2024-11-23T16:14:25,493 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5477215922a748ef8f1ee5c942bf9e3f, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1732378462686 2024-11-23T16:14:25,494 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.Compactor(225): Compacting 547aea4f4f6e4109818f11a11d224b63, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=265, earliestPutTs=1732378463849 2024-11-23T16:14:25,494 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cf5d4648fde06cb520ace381e0be9853:A, priority=-2147483648, current under compaction store size is 1 2024-11-23T16:14:25,494 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T16:14:25,495 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T16:14:25,495 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cf5d4648fde06cb520ace381e0be9853:B, priority=-2147483648, current under compaction store size is 2 2024-11-23T16:14:25,495 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T16:14:25,495 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cf5d4648fde06cb520ace381e0be9853:C, priority=-2147483648, current under compaction store size is 3 2024-11-23T16:14:25,495 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T16:14:25,499 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36997 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T16:14:25,499 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HStore(1541): cf5d4648fde06cb520ace381e0be9853/B is initiating minor compaction (all files) 2024-11-23T16:14:25,499 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cf5d4648fde06cb520ace381e0be9853/B in TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853. 2024-11-23T16:14:25,499 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/B/d6a4fa3d2a37434b9de8157cc23f73bf, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/B/787bc5f7e6e34bf7b0e06e9da46894c1, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/B/e2a75731746a43719653ec1be70a2a2b] into tmpdir=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/.tmp, totalSize=36.1 K 2024-11-23T16:14:25,500 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.Compactor(225): Compacting d6a4fa3d2a37434b9de8157cc23f73bf, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1732378461895 2024-11-23T16:14:25,501 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.Compactor(225): Compacting 787bc5f7e6e34bf7b0e06e9da46894c1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1732378462686 2024-11-23T16:14:25,502 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.Compactor(225): Compacting e2a75731746a43719653ec1be70a2a2b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=265, earliestPutTs=1732378463849 2024-11-23T16:14:25,508 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=cf5d4648fde06cb520ace381e0be9853] 2024-11-23T16:14:25,515 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241123c54ec8bac2c14c228f2ad15b67ac83c5_cf5d4648fde06cb520ace381e0be9853 store=[table=TestAcidGuarantees family=A region=cf5d4648fde06cb520ace381e0be9853] 2024-11-23T16:14:25,518 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241123c54ec8bac2c14c228f2ad15b67ac83c5_cf5d4648fde06cb520ace381e0be9853, store=[table=TestAcidGuarantees family=A region=cf5d4648fde06cb520ace381e0be9853] 2024-11-23T16:14:25,518 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241123c54ec8bac2c14c228f2ad15b67ac83c5_cf5d4648fde06cb520ace381e0be9853 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=cf5d4648fde06cb520ace381e0be9853] 2024-11-23T16:14:25,518 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cf5d4648fde06cb520ace381e0be9853#B#compaction#157 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-23T16:14:25,519 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/.tmp/B/132c7e3c8f734abb862221695a55b037 is 50, key is test_row_0/B:col10/1732378464503/Put/seqid=0 2024-11-23T16:14:25,571 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742012_1188 (size=4469) 2024-11-23T16:14:25,595 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742013_1189 (size=12797) 2024-11-23T16:14:25,604 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/.tmp/B/132c7e3c8f734abb862221695a55b037 as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/B/132c7e3c8f734abb862221695a55b037 2024-11-23T16:14:25,617 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cf5d4648fde06cb520ace381e0be9853/B of cf5d4648fde06cb520ace381e0be9853 into 132c7e3c8f734abb862221695a55b037(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T16:14:25,617 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cf5d4648fde06cb520ace381e0be9853: 2024-11-23T16:14:25,617 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., storeName=cf5d4648fde06cb520ace381e0be9853/B, priority=13, startTime=1732378465495; duration=0sec 2024-11-23T16:14:25,617 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T16:14:25,617 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cf5d4648fde06cb520ace381e0be9853:B 2024-11-23T16:14:25,617 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T16:14:25,619 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36997 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T16:14:25,619 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HStore(1541): cf5d4648fde06cb520ace381e0be9853/C is initiating minor compaction (all files) 2024-11-23T16:14:25,619 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cf5d4648fde06cb520ace381e0be9853/C in TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853. 2024-11-23T16:14:25,619 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/C/deec6a92bb0f4215b405bcfd187e95c9, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/C/4279c0d888624129b950c7373b30fca9, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/C/db8361a1cb654eb9aa52e87c1563755b] into tmpdir=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/.tmp, totalSize=36.1 K 2024-11-23T16:14:25,620 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.Compactor(225): Compacting deec6a92bb0f4215b405bcfd187e95c9, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1732378461895 2024-11-23T16:14:25,620 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.Compactor(225): Compacting 4279c0d888624129b950c7373b30fca9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1732378462686 2024-11-23T16:14:25,620 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.Compactor(225): Compacting db8361a1cb654eb9aa52e87c1563755b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=265, earliestPutTs=1732378463849 2024-11-23T16:14:25,635 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cf5d4648fde06cb520ace381e0be9853#C#compaction#158 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-23T16:14:25,636 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/.tmp/C/2a3cf60e86814ce5b789f782967c3513 is 50, key is test_row_0/C:col10/1732378464503/Put/seqid=0 2024-11-23T16:14:25,641 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-23T16:14:25,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853. 2024-11-23T16:14:25,642 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2902): Flushing cf5d4648fde06cb520ace381e0be9853 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-23T16:14:25,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf5d4648fde06cb520ace381e0be9853, store=A 2024-11-23T16:14:25,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-23T16:14:25,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf5d4648fde06cb520ace381e0be9853, store=B 2024-11-23T16:14:25,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-23T16:14:25,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf5d4648fde06cb520ace381e0be9853, store=C 2024-11-23T16:14:25,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-23T16:14:25,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241123856b072d3dd144d38c42abca4fdfc043_cf5d4648fde06cb520ace381e0be9853 is 50, key is test_row_0/A:col10/1732378464551/Put/seqid=0 2024-11-23T16:14:25,689 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742014_1190 (size=12797) 2024-11-23T16:14:25,692 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853. as already flushing 2024-11-23T16:14:25,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(8855): Flush requested on cf5d4648fde06cb520ace381e0be9853 2024-11-23T16:14:25,696 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/.tmp/C/2a3cf60e86814ce5b789f782967c3513 as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/C/2a3cf60e86814ce5b789f782967c3513 2024-11-23T16:14:25,703 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742015_1191 (size=12454) 2024-11-23T16:14:25,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:25,709 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241123856b072d3dd144d38c42abca4fdfc043_cf5d4648fde06cb520ace381e0be9853 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123856b072d3dd144d38c42abca4fdfc043_cf5d4648fde06cb520ace381e0be9853 2024-11-23T16:14:25,710 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cf5d4648fde06cb520ace381e0be9853/C of cf5d4648fde06cb520ace381e0be9853 into 2a3cf60e86814ce5b789f782967c3513(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T16:14:25,710 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cf5d4648fde06cb520ace381e0be9853: 2024-11-23T16:14:25,710 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., storeName=cf5d4648fde06cb520ace381e0be9853/C, priority=13, startTime=1732378465495; duration=0sec 2024-11-23T16:14:25,710 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T16:14:25,710 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cf5d4648fde06cb520ace381e0be9853:C 2024-11-23T16:14:25,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/.tmp/A/24f9ae4954a445b4804cc0064321f70a, store: [table=TestAcidGuarantees family=A region=cf5d4648fde06cb520ace381e0be9853] 2024-11-23T16:14:25,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/.tmp/A/24f9ae4954a445b4804cc0064321f70a is 175, key is test_row_0/A:col10/1732378464551/Put/seqid=0 2024-11-23T16:14:25,747 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:25,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59586 deadline: 1732378525735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:25,747 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:25,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59638 deadline: 1732378525739, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:25,747 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:25,747 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:25,747 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:14:25,747 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:25,748 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:25,748 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:25,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59628 deadline: 1732378525740, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:25,748 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:14:25,748 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:25,748 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:25,748 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:14:25,758 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742016_1192 (size=31255) 2024-11-23T16:14:25,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-23T16:14:25,865 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:25,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59586 deadline: 1732378525859, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:25,866 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:25,866 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:25,866 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:14:25,867 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:25,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59628 deadline: 1732378525859, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:25,867 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:25,867 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:25,867 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:14:25,879 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:25,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59638 deadline: 1732378525859, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:25,882 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:25,882 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:25,882 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:14:25,973 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): cf5d4648fde06cb520ace381e0be9853#A#compaction#156 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T16:14:25,974 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/.tmp/A/e53a403ff9154cc29a7d893952dbc50b is 175, key is test_row_0/A:col10/1732378464503/Put/seqid=0 2024-11-23T16:14:25,982 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742017_1193 (size=31751) 2024-11-23T16:14:25,992 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/.tmp/A/e53a403ff9154cc29a7d893952dbc50b as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/A/e53a403ff9154cc29a7d893952dbc50b 2024-11-23T16:14:26,005 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cf5d4648fde06cb520ace381e0be9853/A of cf5d4648fde06cb520ace381e0be9853 into e53a403ff9154cc29a7d893952dbc50b(size=31.0 K), total size for store is 31.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T16:14:26,005 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cf5d4648fde06cb520ace381e0be9853: 2024-11-23T16:14:26,005 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., storeName=cf5d4648fde06cb520ace381e0be9853/A, priority=13, startTime=1732378465491; duration=0sec 2024-11-23T16:14:26,005 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T16:14:26,005 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cf5d4648fde06cb520ace381e0be9853:A 2024-11-23T16:14:26,070 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:26,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59586 deadline: 1732378526068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:26,070 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:26,070 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:26,070 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:14:26,081 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:26,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59628 deadline: 1732378526078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:26,081 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:26,081 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:26,081 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:14:26,094 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:26,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59638 deadline: 1732378526088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:26,095 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:26,095 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:26,095 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:14:26,159 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=285, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/.tmp/A/24f9ae4954a445b4804cc0064321f70a 2024-11-23T16:14:26,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/.tmp/B/66abafe5050742da819745e24a2ae177 is 50, key is test_row_0/B:col10/1732378464551/Put/seqid=0 2024-11-23T16:14:26,175 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742018_1194 (size=12301) 2024-11-23T16:14:26,176 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=285 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/.tmp/B/66abafe5050742da819745e24a2ae177 2024-11-23T16:14:26,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/.tmp/C/e7d9fc849d914806960662ebd8e05aef is 50, key is test_row_0/C:col10/1732378464551/Put/seqid=0 2024-11-23T16:14:26,215 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742019_1195 (size=12301) 2024-11-23T16:14:26,219 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=285 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/.tmp/C/e7d9fc849d914806960662ebd8e05aef 2024-11-23T16:14:26,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/.tmp/A/24f9ae4954a445b4804cc0064321f70a as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/A/24f9ae4954a445b4804cc0064321f70a 2024-11-23T16:14:26,231 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/A/24f9ae4954a445b4804cc0064321f70a, entries=150, sequenceid=285, filesize=30.5 K 2024-11-23T16:14:26,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/.tmp/B/66abafe5050742da819745e24a2ae177 as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/B/66abafe5050742da819745e24a2ae177 2024-11-23T16:14:26,238 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/B/66abafe5050742da819745e24a2ae177, entries=150, sequenceid=285, filesize=12.0 K 2024-11-23T16:14:26,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/.tmp/C/e7d9fc849d914806960662ebd8e05aef as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/C/e7d9fc849d914806960662ebd8e05aef 2024-11-23T16:14:26,258 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/C/e7d9fc849d914806960662ebd8e05aef, entries=150, sequenceid=285, filesize=12.0 K 2024-11-23T16:14:26,259 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for cf5d4648fde06cb520ace381e0be9853 in 617ms, sequenceid=285, compaction requested=false 2024-11-23T16:14:26,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2603): Flush status journal for cf5d4648fde06cb520ace381e0be9853: 2024-11-23T16:14:26,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853. 2024-11-23T16:14:26,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=49 2024-11-23T16:14:26,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.HMaster(4169): Remote procedure done, pid=49 2024-11-23T16:14:26,263 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=49, resume processing ppid=48 2024-11-23T16:14:26,263 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=49, ppid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5480 sec 2024-11-23T16:14:26,269 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees in 1.5560 sec 2024-11-23T16:14:26,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(8855): Flush requested on cf5d4648fde06cb520ace381e0be9853 2024-11-23T16:14:26,383 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cf5d4648fde06cb520ace381e0be9853 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-23T16:14:26,383 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf5d4648fde06cb520ace381e0be9853, store=A 2024-11-23T16:14:26,383 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-23T16:14:26,383 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf5d4648fde06cb520ace381e0be9853, store=B 2024-11-23T16:14:26,383 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-23T16:14:26,383 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf5d4648fde06cb520ace381e0be9853, store=C 2024-11-23T16:14:26,383 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-23T16:14:26,396 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241123092ad95a6cf841cabe5693002f0cf765_cf5d4648fde06cb520ace381e0be9853 is 50, key is test_row_0/A:col10/1732378465737/Put/seqid=0 2024-11-23T16:14:26,410 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742020_1196 (size=14994) 2024-11-23T16:14:26,433 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:26,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59586 deadline: 1732378526426, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:26,434 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:26,434 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:26,434 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:14:26,434 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:26,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59638 deadline: 1732378526434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:26,435 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:26,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59628 deadline: 1732378526434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:26,435 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:26,435 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:26,435 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:14:26,436 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:26,436 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:26,436 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:14:26,538 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:26,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59638 deadline: 1732378526538, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:26,539 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:26,539 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:26,539 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:14:26,539 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:26,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59586 deadline: 1732378526538, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:26,540 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:26,540 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:26,540 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:26,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59628 deadline: 1732378526538, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:26,540 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:14:26,540 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:26,540 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:26,540 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:14:26,754 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:26,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59638 deadline: 1732378526748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:26,755 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:26,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59586 deadline: 1732378526749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:26,755 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:26,756 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:26,756 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:26,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59628 deadline: 1732378526749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:26,756 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:14:26,756 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:26,756 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:26,756 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:14:26,757 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:26,757 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:26,757 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:14:26,770 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:26,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59584 deadline: 1732378526768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:26,770 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to 1a7c5470b472,44715,1732378420375 for 'test_row_1' in cf5d4648fde06cb520ace381e0be9853 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8206 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-23T16:14:26,771 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:26,771 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:26,771 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:14:26,811 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:26,815 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241123092ad95a6cf841cabe5693002f0cf765_cf5d4648fde06cb520ace381e0be9853 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123092ad95a6cf841cabe5693002f0cf765_cf5d4648fde06cb520ace381e0be9853 2024-11-23T16:14:26,817 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/.tmp/A/c19e36f81ddb44a9a83b553f1dcbc18e, store: [table=TestAcidGuarantees family=A region=cf5d4648fde06cb520ace381e0be9853] 2024-11-23T16:14:26,818 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/.tmp/A/c19e36f81ddb44a9a83b553f1dcbc18e is 175, key is test_row_0/A:col10/1732378465737/Put/seqid=0 2024-11-23T16:14:26,824 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742021_1197 (size=39949) 2024-11-23T16:14:26,825 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=301, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/.tmp/A/c19e36f81ddb44a9a83b553f1dcbc18e 2024-11-23T16:14:26,836 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/.tmp/B/bee71088d6ba4c8e99248cef4a1e1ca5 is 50, key is test_row_0/B:col10/1732378465737/Put/seqid=0 2024-11-23T16:14:26,841 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742022_1198 (size=12301) 2024-11-23T16:14:26,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-23T16:14:26,868 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-23T16:14:26,869 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-23T16:14:26,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] procedure2.ProcedureExecutor(1139): Stored pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees 2024-11-23T16:14:26,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-23T16:14:26,872 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-23T16:14:26,873 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-23T16:14:26,873 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=51, ppid=50, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-23T16:14:26,898 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:26,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59614 deadline: 1732378526898, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:26,899 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 1a7c5470b472,44715,1732378420375 for 'test_row_1' in cf5d4648fde06cb520ace381e0be9853 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4210 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-23T16:14:26,900 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:26,900 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:26,900 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:14:26,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-23T16:14:27,025 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-23T16:14:27,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853. 2024-11-23T16:14:27,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853. as already flushing 2024-11-23T16:14:27,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853. 2024-11-23T16:14:27,025 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => cf5d4648fde06cb520ace381e0be9853, NAME => 'TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:14:27,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => cf5d4648fde06cb520ace381e0be9853, NAME => 'TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:14:27,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cf5d4648fde06cb520ace381e0be9853, NAME => 'TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cf5d4648fde06cb520ace381e0be9853, NAME => 'TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:14:27,062 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:27,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59586 deadline: 1732378527057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:27,062 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:27,062 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:27,062 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:27,062 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:14:27,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59628 deadline: 1732378527057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:27,063 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:27,063 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:27,063 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:14:27,069 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:27,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59638 deadline: 1732378527068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:27,069 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:27,069 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:27,069 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:14:27,178 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-23T16:14:27,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853. 2024-11-23T16:14:27,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853. as already flushing 2024-11-23T16:14:27,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853. 2024-11-23T16:14:27,179 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => cf5d4648fde06cb520ace381e0be9853, NAME => 'TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:14:27,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => cf5d4648fde06cb520ace381e0be9853, NAME => 'TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:14:27,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cf5d4648fde06cb520ace381e0be9853, NAME => 'TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cf5d4648fde06cb520ace381e0be9853, NAME => 'TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:14:27,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-23T16:14:27,242 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=301 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/.tmp/B/bee71088d6ba4c8e99248cef4a1e1ca5 2024-11-23T16:14:27,250 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/.tmp/C/c17eee5581b0434abce55240de5b4ebb is 50, key is test_row_0/C:col10/1732378465737/Put/seqid=0 2024-11-23T16:14:27,255 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742023_1199 (size=12301) 2024-11-23T16:14:27,331 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-23T16:14:27,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853. 2024-11-23T16:14:27,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853. as already flushing 2024-11-23T16:14:27,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853. 2024-11-23T16:14:27,332 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => cf5d4648fde06cb520ace381e0be9853, NAME => 'TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:14:27,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => cf5d4648fde06cb520ace381e0be9853, NAME => 'TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:14:27,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cf5d4648fde06cb520ace381e0be9853, NAME => 'TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cf5d4648fde06cb520ace381e0be9853, NAME => 'TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:14:27,485 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-23T16:14:27,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853. 2024-11-23T16:14:27,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853. as already flushing 2024-11-23T16:14:27,486 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853. 2024-11-23T16:14:27,486 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => cf5d4648fde06cb520ace381e0be9853, NAME => 'TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:14:27,486 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => cf5d4648fde06cb520ace381e0be9853, NAME => 'TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:14:27,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cf5d4648fde06cb520ace381e0be9853, NAME => 'TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cf5d4648fde06cb520ace381e0be9853, NAME => 'TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:14:27,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-23T16:14:27,570 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:27,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59586 deadline: 1732378527568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:27,570 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:27,571 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:27,571 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:14:27,580 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:27,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59628 deadline: 1732378527578, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:27,581 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:27,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59638 deadline: 1732378527578, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:27,581 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:27,581 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:27,581 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:14:27,581 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:27,581 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:27,581 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:14:27,639 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-23T16:14:27,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853. 2024-11-23T16:14:27,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853. as already flushing 2024-11-23T16:14:27,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853. 2024-11-23T16:14:27,639 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => cf5d4648fde06cb520ace381e0be9853, NAME => 'TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:14:27,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => cf5d4648fde06cb520ace381e0be9853, NAME => 'TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:14:27,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cf5d4648fde06cb520ace381e0be9853, NAME => 'TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cf5d4648fde06cb520ace381e0be9853, NAME => 'TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:14:27,655 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=301 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/.tmp/C/c17eee5581b0434abce55240de5b4ebb 2024-11-23T16:14:27,662 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/.tmp/A/c19e36f81ddb44a9a83b553f1dcbc18e as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/A/c19e36f81ddb44a9a83b553f1dcbc18e 2024-11-23T16:14:27,666 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/A/c19e36f81ddb44a9a83b553f1dcbc18e, entries=200, sequenceid=301, filesize=39.0 K 2024-11-23T16:14:27,667 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/.tmp/B/bee71088d6ba4c8e99248cef4a1e1ca5 as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/B/bee71088d6ba4c8e99248cef4a1e1ca5 2024-11-23T16:14:27,677 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/B/bee71088d6ba4c8e99248cef4a1e1ca5, entries=150, sequenceid=301, filesize=12.0 K 2024-11-23T16:14:27,678 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/.tmp/C/c17eee5581b0434abce55240de5b4ebb as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/C/c17eee5581b0434abce55240de5b4ebb 2024-11-23T16:14:27,683 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/C/c17eee5581b0434abce55240de5b4ebb, entries=150, sequenceid=301, filesize=12.0 K 2024-11-23T16:14:27,684 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=107.34 KB/109920 for cf5d4648fde06cb520ace381e0be9853 in 1301ms, sequenceid=301, compaction requested=true 2024-11-23T16:14:27,684 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cf5d4648fde06cb520ace381e0be9853: 2024-11-23T16:14:27,684 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T16:14:27,685 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cf5d4648fde06cb520ace381e0be9853:A, priority=-2147483648, current under compaction store size is 1 2024-11-23T16:14:27,685 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T16:14:27,685 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T16:14:27,685 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cf5d4648fde06cb520ace381e0be9853:B, priority=-2147483648, current under compaction store size is 2 2024-11-23T16:14:27,685 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T16:14:27,685 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cf5d4648fde06cb520ace381e0be9853:C, priority=-2147483648, current under compaction store size is 3 2024-11-23T16:14:27,685 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T16:14:27,686 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102955 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T16:14:27,686 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HStore(1541): cf5d4648fde06cb520ace381e0be9853/A is initiating minor compaction (all files) 2024-11-23T16:14:27,686 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cf5d4648fde06cb520ace381e0be9853/A in TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853. 2024-11-23T16:14:27,686 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37399 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T16:14:27,686 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/A/e53a403ff9154cc29a7d893952dbc50b, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/A/24f9ae4954a445b4804cc0064321f70a, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/A/c19e36f81ddb44a9a83b553f1dcbc18e] into tmpdir=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/.tmp, totalSize=100.5 K 2024-11-23T16:14:27,686 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HStore(1541): cf5d4648fde06cb520ace381e0be9853/B is initiating minor compaction (all files) 2024-11-23T16:14:27,686 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853. 2024-11-23T16:14:27,686 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cf5d4648fde06cb520ace381e0be9853/B in TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853. 2024-11-23T16:14:27,686 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853. files: [hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/A/e53a403ff9154cc29a7d893952dbc50b, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/A/24f9ae4954a445b4804cc0064321f70a, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/A/c19e36f81ddb44a9a83b553f1dcbc18e] 2024-11-23T16:14:27,686 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/B/132c7e3c8f734abb862221695a55b037, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/B/66abafe5050742da819745e24a2ae177, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/B/bee71088d6ba4c8e99248cef4a1e1ca5] into tmpdir=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/.tmp, totalSize=36.5 K 2024-11-23T16:14:27,687 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.Compactor(225): Compacting e53a403ff9154cc29a7d893952dbc50b, keycount=150, bloomtype=ROW, size=31.0 K, encoding=NONE, compression=NONE, seqNum=265, earliestPutTs=1732378463849 2024-11-23T16:14:27,687 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.Compactor(225): Compacting 132c7e3c8f734abb862221695a55b037, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=265, earliestPutTs=1732378463849 2024-11-23T16:14:27,688 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.Compactor(225): Compacting 66abafe5050742da819745e24a2ae177, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=285, earliestPutTs=1732378464541 2024-11-23T16:14:27,688 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.Compactor(225): Compacting 24f9ae4954a445b4804cc0064321f70a, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=285, earliestPutTs=1732378464541 2024-11-23T16:14:27,688 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.Compactor(225): Compacting bee71088d6ba4c8e99248cef4a1e1ca5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=301, earliestPutTs=1732378465737 2024-11-23T16:14:27,689 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.Compactor(225): Compacting c19e36f81ddb44a9a83b553f1dcbc18e, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=301, earliestPutTs=1732378465733 2024-11-23T16:14:27,701 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=cf5d4648fde06cb520ace381e0be9853] 2024-11-23T16:14:27,705 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cf5d4648fde06cb520ace381e0be9853#B#compaction#166 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-23T16:14:27,705 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241123107606b854a7453bb60e90ad4f6c6c3f_cf5d4648fde06cb520ace381e0be9853 store=[table=TestAcidGuarantees family=A region=cf5d4648fde06cb520ace381e0be9853] 2024-11-23T16:14:27,705 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/.tmp/B/cb03383e6607400c8e097af59514a54a is 50, key is test_row_0/B:col10/1732378465737/Put/seqid=0 2024-11-23T16:14:27,707 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241123107606b854a7453bb60e90ad4f6c6c3f_cf5d4648fde06cb520ace381e0be9853, store=[table=TestAcidGuarantees family=A region=cf5d4648fde06cb520ace381e0be9853] 2024-11-23T16:14:27,707 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241123107606b854a7453bb60e90ad4f6c6c3f_cf5d4648fde06cb520ace381e0be9853 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=cf5d4648fde06cb520ace381e0be9853] 2024-11-23T16:14:27,723 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742025_1201 (size=4469) 2024-11-23T16:14:27,725 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742024_1200 (size=12949) 2024-11-23T16:14:27,792 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-23T16:14:27,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853. 2024-11-23T16:14:27,793 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2902): Flushing cf5d4648fde06cb520ace381e0be9853 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-23T16:14:27,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf5d4648fde06cb520ace381e0be9853, store=A 2024-11-23T16:14:27,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-23T16:14:27,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf5d4648fde06cb520ace381e0be9853, store=B 2024-11-23T16:14:27,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-23T16:14:27,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf5d4648fde06cb520ace381e0be9853, store=C 2024-11-23T16:14:27,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-23T16:14:27,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411234c8ac4c07c6d4aeeb6d98599c6c922d5_cf5d4648fde06cb520ace381e0be9853 is 50, key is test_row_0/A:col10/1732378466430/Put/seqid=0 2024-11-23T16:14:27,810 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742026_1202 (size=12454) 2024-11-23T16:14:28,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-23T16:14:28,125 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): cf5d4648fde06cb520ace381e0be9853#A#compaction#165 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T16:14:28,126 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/.tmp/A/1d08693d212c422786dfeceb72435999 is 175, key is test_row_0/A:col10/1732378465737/Put/seqid=0 2024-11-23T16:14:28,134 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/.tmp/B/cb03383e6607400c8e097af59514a54a as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/B/cb03383e6607400c8e097af59514a54a 2024-11-23T16:14:28,142 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cf5d4648fde06cb520ace381e0be9853/B of cf5d4648fde06cb520ace381e0be9853 into cb03383e6607400c8e097af59514a54a(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T16:14:28,142 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cf5d4648fde06cb520ace381e0be9853: 2024-11-23T16:14:28,142 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., storeName=cf5d4648fde06cb520ace381e0be9853/B, priority=13, startTime=1732378467685; duration=0sec 2024-11-23T16:14:28,143 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T16:14:28,143 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cf5d4648fde06cb520ace381e0be9853:B 2024-11-23T16:14:28,144 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T16:14:28,146 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37399 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T16:14:28,146 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HStore(1541): cf5d4648fde06cb520ace381e0be9853/C is initiating minor compaction (all files) 2024-11-23T16:14:28,146 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cf5d4648fde06cb520ace381e0be9853/C in TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853. 2024-11-23T16:14:28,147 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/C/2a3cf60e86814ce5b789f782967c3513, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/C/e7d9fc849d914806960662ebd8e05aef, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/C/c17eee5581b0434abce55240de5b4ebb] into tmpdir=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/.tmp, totalSize=36.5 K 2024-11-23T16:14:28,147 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.Compactor(225): Compacting 2a3cf60e86814ce5b789f782967c3513, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=265, earliestPutTs=1732378463849 2024-11-23T16:14:28,148 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.Compactor(225): Compacting e7d9fc849d914806960662ebd8e05aef, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=285, earliestPutTs=1732378464541 2024-11-23T16:14:28,148 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.Compactor(225): Compacting c17eee5581b0434abce55240de5b4ebb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=301, earliestPutTs=1732378465737 2024-11-23T16:14:28,154 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742027_1203 (size=31903) 2024-11-23T16:14:28,164 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cf5d4648fde06cb520ace381e0be9853#C#compaction#168 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T16:14:28,165 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/.tmp/C/410b96d8455449598ed19d2ed3f23b89 is 50, key is test_row_0/C:col10/1732378465737/Put/seqid=0 2024-11-23T16:14:28,183 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742028_1204 (size=12949) 2024-11-23T16:14:28,193 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/.tmp/C/410b96d8455449598ed19d2ed3f23b89 as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/C/410b96d8455449598ed19d2ed3f23b89 2024-11-23T16:14:28,202 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cf5d4648fde06cb520ace381e0be9853/C of cf5d4648fde06cb520ace381e0be9853 into 410b96d8455449598ed19d2ed3f23b89(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T16:14:28,202 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cf5d4648fde06cb520ace381e0be9853: 2024-11-23T16:14:28,202 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., storeName=cf5d4648fde06cb520ace381e0be9853/C, priority=13, startTime=1732378467685; duration=0sec 2024-11-23T16:14:28,202 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T16:14:28,202 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cf5d4648fde06cb520ace381e0be9853:C 2024-11-23T16:14:28,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:28,217 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411234c8ac4c07c6d4aeeb6d98599c6c922d5_cf5d4648fde06cb520ace381e0be9853 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411234c8ac4c07c6d4aeeb6d98599c6c922d5_cf5d4648fde06cb520ace381e0be9853 2024-11-23T16:14:28,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/.tmp/A/a3966beaa6f349b88e01530018370935, store: [table=TestAcidGuarantees family=A region=cf5d4648fde06cb520ace381e0be9853] 2024-11-23T16:14:28,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/.tmp/A/a3966beaa6f349b88e01530018370935 is 175, key is test_row_0/A:col10/1732378466430/Put/seqid=0 2024-11-23T16:14:28,225 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742029_1205 (size=31255) 2024-11-23T16:14:28,561 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/.tmp/A/1d08693d212c422786dfeceb72435999 as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/A/1d08693d212c422786dfeceb72435999 2024-11-23T16:14:28,567 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cf5d4648fde06cb520ace381e0be9853/A of cf5d4648fde06cb520ace381e0be9853 into 1d08693d212c422786dfeceb72435999(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T16:14:28,567 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cf5d4648fde06cb520ace381e0be9853: 2024-11-23T16:14:28,567 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., storeName=cf5d4648fde06cb520ace381e0be9853/A, priority=13, startTime=1732378467684; duration=0sec 2024-11-23T16:14:28,567 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T16:14:28,567 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cf5d4648fde06cb520ace381e0be9853:A 2024-11-23T16:14:28,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(8855): Flush requested on cf5d4648fde06cb520ace381e0be9853 2024-11-23T16:14:28,581 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853. as already flushing 2024-11-23T16:14:28,605 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:28,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59638 deadline: 1732378528600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:28,607 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:28,607 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:28,607 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:14:28,608 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:28,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59586 deadline: 1732378528603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:28,609 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:28,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59628 deadline: 1732378528604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:28,609 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:28,609 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:28,609 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:14:28,616 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:28,616 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:28,616 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:14:28,627 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=320, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/.tmp/A/a3966beaa6f349b88e01530018370935 2024-11-23T16:14:28,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/.tmp/B/66dee4c6ddbf4593a032fca6ec7c2e6b is 50, key is test_row_0/B:col10/1732378466430/Put/seqid=0 2024-11-23T16:14:28,681 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742030_1206 (size=12301) 2024-11-23T16:14:28,681 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=320 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/.tmp/B/66dee4c6ddbf4593a032fca6ec7c2e6b 2024-11-23T16:14:28,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/.tmp/C/fb9d1f5f1e1e4fd8a3dd40cbe8dc2796 is 50, key is test_row_0/C:col10/1732378466430/Put/seqid=0 2024-11-23T16:14:28,712 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742031_1207 (size=12301) 2024-11-23T16:14:28,725 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:28,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59628 deadline: 1732378528718, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:28,726 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:28,726 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:28,726 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:14:28,726 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:28,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59586 deadline: 1732378528718, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:28,727 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:28,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59638 deadline: 1732378528718, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:28,727 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:28,728 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:28,728 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:14:28,728 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:28,728 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:28,728 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:14:28,929 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:28,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59628 deadline: 1732378528928, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:28,929 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:28,929 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:28,930 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:14:28,941 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:28,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59638 deadline: 1732378528938, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:28,942 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:28,942 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:28,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59586 deadline: 1732378528938, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:28,942 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:28,942 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:14:28,942 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:28,942 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:28,943 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:14:29,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-23T16:14:29,113 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=320 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/.tmp/C/fb9d1f5f1e1e4fd8a3dd40cbe8dc2796 2024-11-23T16:14:29,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/.tmp/A/a3966beaa6f349b88e01530018370935 as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/A/a3966beaa6f349b88e01530018370935 2024-11-23T16:14:29,139 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/A/a3966beaa6f349b88e01530018370935, entries=150, sequenceid=320, filesize=30.5 K 2024-11-23T16:14:29,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/.tmp/B/66dee4c6ddbf4593a032fca6ec7c2e6b as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/B/66dee4c6ddbf4593a032fca6ec7c2e6b 2024-11-23T16:14:29,146 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/B/66dee4c6ddbf4593a032fca6ec7c2e6b, entries=150, sequenceid=320, filesize=12.0 K 2024-11-23T16:14:29,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/.tmp/C/fb9d1f5f1e1e4fd8a3dd40cbe8dc2796 as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/C/fb9d1f5f1e1e4fd8a3dd40cbe8dc2796 2024-11-23T16:14:29,152 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/C/fb9d1f5f1e1e4fd8a3dd40cbe8dc2796, entries=150, sequenceid=320, filesize=12.0 K 2024-11-23T16:14:29,153 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for cf5d4648fde06cb520ace381e0be9853 in 1360ms, sequenceid=320, compaction requested=false 2024-11-23T16:14:29,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2603): Flush status journal for cf5d4648fde06cb520ace381e0be9853: 2024-11-23T16:14:29,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853. 2024-11-23T16:14:29,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=51 2024-11-23T16:14:29,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.HMaster(4169): Remote procedure done, pid=51 2024-11-23T16:14:29,158 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=51, resume processing ppid=50 2024-11-23T16:14:29,158 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=51, ppid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2820 sec 2024-11-23T16:14:29,160 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees in 2.2900 sec 2024-11-23T16:14:29,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(8855): Flush requested on cf5d4648fde06cb520ace381e0be9853 2024-11-23T16:14:29,241 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cf5d4648fde06cb520ace381e0be9853 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-23T16:14:29,241 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf5d4648fde06cb520ace381e0be9853, store=A 2024-11-23T16:14:29,241 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-23T16:14:29,241 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf5d4648fde06cb520ace381e0be9853, store=B 2024-11-23T16:14:29,241 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-23T16:14:29,241 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf5d4648fde06cb520ace381e0be9853, store=C 2024-11-23T16:14:29,241 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-23T16:14:29,255 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241123d6711afba503466995afc6582c60620f_cf5d4648fde06cb520ace381e0be9853 is 50, key is test_row_0/A:col10/1732378469239/Put/seqid=0 2024-11-23T16:14:29,280 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742032_1208 (size=14994) 2024-11-23T16:14:29,281 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:29,286 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241123d6711afba503466995afc6582c60620f_cf5d4648fde06cb520ace381e0be9853 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123d6711afba503466995afc6582c60620f_cf5d4648fde06cb520ace381e0be9853 2024-11-23T16:14:29,287 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:29,287 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/.tmp/A/0f7dc6b491da459bb539c172c9e15d81, store: [table=TestAcidGuarantees family=A region=cf5d4648fde06cb520ace381e0be9853] 2024-11-23T16:14:29,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59586 deadline: 1732378529284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:29,288 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:29,288 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:29,288 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:14:29,288 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/.tmp/A/0f7dc6b491da459bb539c172c9e15d81 is 175, key is test_row_0/A:col10/1732378469239/Put/seqid=0 2024-11-23T16:14:29,293 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:29,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59628 deadline: 1732378529288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:29,293 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:29,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59638 deadline: 1732378529289, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:29,294 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:29,294 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:29,294 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:14:29,296 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:29,296 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:29,296 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:14:29,302 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742033_1209 (size=39949) 2024-11-23T16:14:29,302 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=337, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/.tmp/A/0f7dc6b491da459bb539c172c9e15d81 2024-11-23T16:14:29,313 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/.tmp/B/1174617488a94552afa9e1a3694e44a1 is 50, key is test_row_0/B:col10/1732378469239/Put/seqid=0 2024-11-23T16:14:29,320 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742034_1210 (size=12301) 2024-11-23T16:14:29,402 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:29,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59586 deadline: 1732378529398, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:29,403 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:29,403 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:29,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59638 deadline: 1732378529398, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:29,403 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:29,403 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:14:29,403 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:29,403 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:29,403 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:29,403 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:14:29,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59628 deadline: 1732378529398, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:29,404 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:29,404 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:29,404 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:14:29,613 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:29,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59586 deadline: 1732378529608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:29,613 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:29,613 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:29,613 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:29,613 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:14:29,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59628 deadline: 1732378529609, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:29,614 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:29,614 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:29,614 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:29,614 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:14:29,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59638 deadline: 1732378529609, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:29,614 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:29,615 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:29,615 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:14:29,721 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=337 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/.tmp/B/1174617488a94552afa9e1a3694e44a1 2024-11-23T16:14:29,730 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/.tmp/C/0663c9ac1a9140369c8e278c38a74468 is 50, key is test_row_0/C:col10/1732378469239/Put/seqid=0 2024-11-23T16:14:29,738 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742035_1211 (size=12301) 2024-11-23T16:14:29,920 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:29,920 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:29,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59638 deadline: 1732378529918, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:29,921 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:29,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59586 deadline: 1732378529918, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:29,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59628 deadline: 1732378529918, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:29,921 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:29,921 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:29,921 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:29,921 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:14:29,921 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:29,921 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:29,921 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:14:29,921 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:29,921 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:14:30,139 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=337 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/.tmp/C/0663c9ac1a9140369c8e278c38a74468 2024-11-23T16:14:30,147 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/.tmp/A/0f7dc6b491da459bb539c172c9e15d81 as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/A/0f7dc6b491da459bb539c172c9e15d81 2024-11-23T16:14:30,152 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/A/0f7dc6b491da459bb539c172c9e15d81, entries=200, sequenceid=337, filesize=39.0 K 2024-11-23T16:14:30,153 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/.tmp/B/1174617488a94552afa9e1a3694e44a1 as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/B/1174617488a94552afa9e1a3694e44a1 2024-11-23T16:14:30,158 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/B/1174617488a94552afa9e1a3694e44a1, entries=150, sequenceid=337, filesize=12.0 K 2024-11-23T16:14:30,162 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/.tmp/C/0663c9ac1a9140369c8e278c38a74468 as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/C/0663c9ac1a9140369c8e278c38a74468 2024-11-23T16:14:30,169 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/C/0663c9ac1a9140369c8e278c38a74468, entries=150, sequenceid=337, filesize=12.0 K 2024-11-23T16:14:30,170 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=107.34 KB/109920 for cf5d4648fde06cb520ace381e0be9853 in 929ms, sequenceid=337, compaction requested=true 2024-11-23T16:14:30,170 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cf5d4648fde06cb520ace381e0be9853: 2024-11-23T16:14:30,170 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cf5d4648fde06cb520ace381e0be9853:A, priority=-2147483648, current under compaction store size is 1 2024-11-23T16:14:30,171 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T16:14:30,171 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cf5d4648fde06cb520ace381e0be9853:B, priority=-2147483648, current under compaction store size is 2 2024-11-23T16:14:30,171 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-23T16:14:30,171 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cf5d4648fde06cb520ace381e0be9853:C, priority=-2147483648, current under compaction store size is 3 2024-11-23T16:14:30,171 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-11-23T16:14:30,171 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T16:14:30,175 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103107 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T16:14:30,176 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HStore(1541): cf5d4648fde06cb520ace381e0be9853/A is initiating minor compaction (all files) 2024-11-23T16:14:30,176 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cf5d4648fde06cb520ace381e0be9853/A in TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853. 2024-11-23T16:14:30,176 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/A/1d08693d212c422786dfeceb72435999, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/A/a3966beaa6f349b88e01530018370935, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/A/0f7dc6b491da459bb539c172c9e15d81] into tmpdir=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/.tmp, totalSize=100.7 K 2024-11-23T16:14:30,176 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853. 2024-11-23T16:14:30,176 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853. files: [hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/A/1d08693d212c422786dfeceb72435999, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/A/a3966beaa6f349b88e01530018370935, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/A/0f7dc6b491da459bb539c172c9e15d81] 2024-11-23T16:14:30,176 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T16:14:30,176 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1d08693d212c422786dfeceb72435999, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=301, earliestPutTs=1732378465737 2024-11-23T16:14:30,178 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.Compactor(225): Compacting a3966beaa6f349b88e01530018370935, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=320, earliestPutTs=1732378466403 2024-11-23T16:14:30,178 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0f7dc6b491da459bb539c172c9e15d81, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=337, earliestPutTs=1732378468584 2024-11-23T16:14:30,178 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T16:14:30,178 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HStore(1541): cf5d4648fde06cb520ace381e0be9853/C is initiating minor compaction (all files) 2024-11-23T16:14:30,178 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cf5d4648fde06cb520ace381e0be9853/C in TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853. 2024-11-23T16:14:30,179 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/C/410b96d8455449598ed19d2ed3f23b89, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/C/fb9d1f5f1e1e4fd8a3dd40cbe8dc2796, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/C/0663c9ac1a9140369c8e278c38a74468] into tmpdir=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/.tmp, totalSize=36.7 K 2024-11-23T16:14:30,179 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.Compactor(225): Compacting 410b96d8455449598ed19d2ed3f23b89, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=301, earliestPutTs=1732378465737 2024-11-23T16:14:30,180 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.Compactor(225): Compacting fb9d1f5f1e1e4fd8a3dd40cbe8dc2796, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=320, earliestPutTs=1732378466403 2024-11-23T16:14:30,188 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.Compactor(225): Compacting 0663c9ac1a9140369c8e278c38a74468, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=337, earliestPutTs=1732378468588 2024-11-23T16:14:30,201 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=cf5d4648fde06cb520ace381e0be9853] 2024-11-23T16:14:30,203 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411237f9f864d6df54666b47e977417e0f19d_cf5d4648fde06cb520ace381e0be9853 store=[table=TestAcidGuarantees family=A region=cf5d4648fde06cb520ace381e0be9853] 2024-11-23T16:14:30,204 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cf5d4648fde06cb520ace381e0be9853#C#compaction#175 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-23T16:14:30,205 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/.tmp/C/84dc627c32b948c9aaf05a80aa818018 is 50, key is test_row_0/C:col10/1732378469239/Put/seqid=0 2024-11-23T16:14:30,205 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411237f9f864d6df54666b47e977417e0f19d_cf5d4648fde06cb520ace381e0be9853, store=[table=TestAcidGuarantees family=A region=cf5d4648fde06cb520ace381e0be9853] 2024-11-23T16:14:30,206 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411237f9f864d6df54666b47e977417e0f19d_cf5d4648fde06cb520ace381e0be9853 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=cf5d4648fde06cb520ace381e0be9853] 2024-11-23T16:14:30,219 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742037_1213 (size=4469) 2024-11-23T16:14:30,221 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): cf5d4648fde06cb520ace381e0be9853#A#compaction#174 average throughput is 1.29 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T16:14:30,221 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/.tmp/A/2bdf1def87fc4ffca6fac4d59284cd8f is 175, key is test_row_0/A:col10/1732378469239/Put/seqid=0 2024-11-23T16:14:30,229 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742036_1212 (size=13051) 2024-11-23T16:14:30,236 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/.tmp/C/84dc627c32b948c9aaf05a80aa818018 as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/C/84dc627c32b948c9aaf05a80aa818018 2024-11-23T16:14:30,244 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cf5d4648fde06cb520ace381e0be9853/C of cf5d4648fde06cb520ace381e0be9853 into 84dc627c32b948c9aaf05a80aa818018(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T16:14:30,244 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cf5d4648fde06cb520ace381e0be9853: 2024-11-23T16:14:30,244 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., storeName=cf5d4648fde06cb520ace381e0be9853/C, priority=13, startTime=1732378470171; duration=0sec 2024-11-23T16:14:30,244 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T16:14:30,244 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cf5d4648fde06cb520ace381e0be9853:C 2024-11-23T16:14:30,244 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T16:14:30,246 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T16:14:30,246 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HStore(1541): cf5d4648fde06cb520ace381e0be9853/B is initiating minor compaction (all files) 2024-11-23T16:14:30,246 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cf5d4648fde06cb520ace381e0be9853/B in TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853. 2024-11-23T16:14:30,246 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/B/cb03383e6607400c8e097af59514a54a, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/B/66dee4c6ddbf4593a032fca6ec7c2e6b, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/B/1174617488a94552afa9e1a3694e44a1] into tmpdir=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/.tmp, totalSize=36.7 K 2024-11-23T16:14:30,247 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.Compactor(225): Compacting cb03383e6607400c8e097af59514a54a, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=301, earliestPutTs=1732378465737 2024-11-23T16:14:30,248 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.Compactor(225): Compacting 66dee4c6ddbf4593a032fca6ec7c2e6b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=320, earliestPutTs=1732378466403 2024-11-23T16:14:30,249 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.Compactor(225): Compacting 1174617488a94552afa9e1a3694e44a1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=337, earliestPutTs=1732378468588 2024-11-23T16:14:30,258 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cf5d4648fde06cb520ace381e0be9853#B#compaction#176 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T16:14:30,259 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/.tmp/B/01c7c21da7c244dfaa73f9591258d0be is 50, key is test_row_0/B:col10/1732378469239/Put/seqid=0 2024-11-23T16:14:30,265 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742038_1214 (size=32005) 2024-11-23T16:14:30,274 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/.tmp/A/2bdf1def87fc4ffca6fac4d59284cd8f as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/A/2bdf1def87fc4ffca6fac4d59284cd8f 2024-11-23T16:14:30,283 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742039_1215 (size=13051) 2024-11-23T16:14:30,287 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cf5d4648fde06cb520ace381e0be9853/A of cf5d4648fde06cb520ace381e0be9853 into 2bdf1def87fc4ffca6fac4d59284cd8f(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T16:14:30,287 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cf5d4648fde06cb520ace381e0be9853: 2024-11-23T16:14:30,287 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., storeName=cf5d4648fde06cb520ace381e0be9853/A, priority=13, startTime=1732378470170; duration=0sec 2024-11-23T16:14:30,287 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T16:14:30,287 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cf5d4648fde06cb520ace381e0be9853:A 2024-11-23T16:14:30,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(8855): Flush requested on cf5d4648fde06cb520ace381e0be9853 2024-11-23T16:14:30,434 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cf5d4648fde06cb520ace381e0be9853 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-23T16:14:30,435 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf5d4648fde06cb520ace381e0be9853, store=A 2024-11-23T16:14:30,435 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-23T16:14:30,435 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf5d4648fde06cb520ace381e0be9853, store=B 2024-11-23T16:14:30,435 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-23T16:14:30,436 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf5d4648fde06cb520ace381e0be9853, store=C 2024-11-23T16:14:30,436 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-23T16:14:30,454 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:30,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59586 deadline: 1732378530447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:30,454 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:30,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59628 deadline: 1732378530449, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:30,454 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:30,454 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:30,454 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411231de759dc86364cf691ee759b4d9ccec4_cf5d4648fde06cb520ace381e0be9853 is 50, key is test_row_0/A:col10/1732378469284/Put/seqid=0 2024-11-23T16:14:30,454 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:14:30,455 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:30,455 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:30,455 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:14:30,459 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:30,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59638 deadline: 1732378530454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:30,459 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:30,459 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:30,459 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:14:30,478 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742040_1216 (size=17534) 2024-11-23T16:14:30,563 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:30,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59586 deadline: 1732378530558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:30,563 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:30,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59628 deadline: 1732378530558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:30,563 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:30,563 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:30,563 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:14:30,563 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:30,563 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:30,563 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:14:30,568 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:30,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59638 deadline: 1732378530568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:30,568 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:30,568 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:30,568 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:14:30,689 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/.tmp/B/01c7c21da7c244dfaa73f9591258d0be as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/B/01c7c21da7c244dfaa73f9591258d0be 2024-11-23T16:14:30,694 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cf5d4648fde06cb520ace381e0be9853/B of cf5d4648fde06cb520ace381e0be9853 into 01c7c21da7c244dfaa73f9591258d0be(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T16:14:30,694 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cf5d4648fde06cb520ace381e0be9853: 2024-11-23T16:14:30,694 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., storeName=cf5d4648fde06cb520ace381e0be9853/B, priority=13, startTime=1732378470171; duration=0sec 2024-11-23T16:14:30,694 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T16:14:30,694 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cf5d4648fde06cb520ace381e0be9853:B 2024-11-23T16:14:30,771 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:30,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59586 deadline: 1732378530768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:30,772 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:30,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59628 deadline: 1732378530768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:30,772 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:30,772 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:30,772 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:14:30,772 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:30,772 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:30,772 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:14:30,778 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:30,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59638 deadline: 1732378530778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:30,779 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:30,779 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:30,779 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:14:30,879 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:30,884 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411231de759dc86364cf691ee759b4d9ccec4_cf5d4648fde06cb520ace381e0be9853 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411231de759dc86364cf691ee759b4d9ccec4_cf5d4648fde06cb520ace381e0be9853 2024-11-23T16:14:30,885 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/.tmp/A/0e66cbdcb82d41e4ae53bd3397947e6c, store: [table=TestAcidGuarantees family=A region=cf5d4648fde06cb520ace381e0be9853] 2024-11-23T16:14:30,885 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/.tmp/A/0e66cbdcb82d41e4ae53bd3397947e6c is 175, key is test_row_0/A:col10/1732378469284/Put/seqid=0 2024-11-23T16:14:30,889 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742041_1217 (size=48639) 2024-11-23T16:14:30,889 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=360, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/.tmp/A/0e66cbdcb82d41e4ae53bd3397947e6c 2024-11-23T16:14:30,898 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/.tmp/B/9b7fef690e4449549e61a0f00cca7b9f is 50, key is test_row_0/B:col10/1732378469284/Put/seqid=0 2024-11-23T16:14:30,906 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742042_1218 (size=12301) 2024-11-23T16:14:30,907 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=360 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/.tmp/B/9b7fef690e4449549e61a0f00cca7b9f 2024-11-23T16:14:30,915 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/.tmp/C/a74a14f34a3542adb828cfefbebf05d9 is 50, key is test_row_0/C:col10/1732378469284/Put/seqid=0 2024-11-23T16:14:30,919 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742043_1219 (size=12301) 2024-11-23T16:14:30,925 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=360 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/.tmp/C/a74a14f34a3542adb828cfefbebf05d9 2024-11-23T16:14:30,930 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/.tmp/A/0e66cbdcb82d41e4ae53bd3397947e6c as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/A/0e66cbdcb82d41e4ae53bd3397947e6c 2024-11-23T16:14:30,935 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/A/0e66cbdcb82d41e4ae53bd3397947e6c, entries=250, sequenceid=360, filesize=47.5 K 2024-11-23T16:14:30,936 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/.tmp/B/9b7fef690e4449549e61a0f00cca7b9f as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/B/9b7fef690e4449549e61a0f00cca7b9f 2024-11-23T16:14:30,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:30,940 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/B/9b7fef690e4449549e61a0f00cca7b9f, entries=150, sequenceid=360, filesize=12.0 K 2024-11-23T16:14:30,941 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/.tmp/C/a74a14f34a3542adb828cfefbebf05d9 as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/C/a74a14f34a3542adb828cfefbebf05d9 2024-11-23T16:14:30,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:30,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:30,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:30,947 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/C/a74a14f34a3542adb828cfefbebf05d9, entries=150, sequenceid=360, filesize=12.0 K 2024-11-23T16:14:30,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:30,948 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=53.67 KB/54960 for cf5d4648fde06cb520ace381e0be9853 in 514ms, sequenceid=360, compaction requested=false 2024-11-23T16:14:30,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:30,948 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cf5d4648fde06cb520ace381e0be9853: 2024-11-23T16:14:30,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:30,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:30,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:30,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:30,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:30,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:30,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:30,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:30,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:30,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:30,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:30,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:30,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:30,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:30,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:30,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:30,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:30,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:30,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:30,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:30,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:30,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:30,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:30,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:30,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:30,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:30,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:30,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:30,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:30,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:30,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:30,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:30,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:30,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:30,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:30,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:30,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:30,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:30,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:30,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:30,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:30,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:30,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:30,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:30,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:30,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:30,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:30,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:30,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:30,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:30,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:30,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:30,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:30,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:30,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:30,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:30,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:30,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:30,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:30,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:30,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:30,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:30,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:30,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:30,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:30,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:30,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:30,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:30,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:30,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:30,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:30,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:30,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:30,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:30,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:30,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:30,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:30,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:30,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:30,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:30,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:30,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:30,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:30,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:30,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:30,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:30,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:30,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:30,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:30,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:30,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:30,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:30,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:30,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:30,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:30,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:30,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:30,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:30,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:30,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:30,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:30,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:30,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:30,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:30,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:30,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:30,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:30,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:30,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:30,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:30,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:30,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:30,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:30,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:30,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:30,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:30,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:30,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:30,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:30,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:30,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:30,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:30,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:30,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:30,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:30,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:30,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:30,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:30,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:30,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:31,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:31,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:31,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:31,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:31,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:31,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:31,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:31,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:31,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:31,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:31,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:31,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:31,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:31,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:31,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(8855): Flush requested on cf5d4648fde06cb520ace381e0be9853 2024-11-23T16:14:31,013 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cf5d4648fde06cb520ace381e0be9853 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-23T16:14:31,014 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf5d4648fde06cb520ace381e0be9853, store=A 2024-11-23T16:14:31,014 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-23T16:14:31,014 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf5d4648fde06cb520ace381e0be9853, store=B 2024-11-23T16:14:31,014 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-23T16:14:31,014 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf5d4648fde06cb520ace381e0be9853, store=C 2024-11-23T16:14:31,014 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-23T16:14:31,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:31,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:31,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:31,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:31,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:31,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:31,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:31,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:31,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:31,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:31,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:31,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:31,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:31,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:31,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:31,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:31,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:31,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:31,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:31,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:31,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:31,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:31,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:31,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:31,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:31,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:31,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:31,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:31,033 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112309de6b870b6c4a11b02106a02e0c0ae8_cf5d4648fde06cb520ace381e0be9853 is 50, key is test_row_1/A:col10/1732378471012/Put/seqid=0 2024-11-23T16:14:31,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:31,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:31,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:31,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:31,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:31,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:31,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:31,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:31,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:31,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:31,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:31,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:31,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:31,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:31,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:31,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:31,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:31,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:31,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:31,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:31,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:31,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:31,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:31,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:31,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:31,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:31,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:31,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:31,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:31,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:31,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:31,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:31,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:31,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:31,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:31,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:31,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:31,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:31,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:31,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:31,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:31,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:31,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:31,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:31,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:31,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:31,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:31,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:31,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:31,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:31,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-23T16:14:31,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:31,048 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-23T16:14:31,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:31,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:31,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:31,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:31,050 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-23T16:14:31,050 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742044_1220 (size=12454) 2024-11-23T16:14:31,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:31,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] procedure2.ProcedureExecutor(1139): Stored pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees 2024-11-23T16:14:31,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:31,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-23T16:14:31,052 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-23T16:14:31,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:31,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:31,053 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-23T16:14:31,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:31,053 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=53, ppid=52, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-23T16:14:31,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:31,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:31,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:31,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:31,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:31,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:31,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:31,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:31,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:31,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:31,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:31,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:31,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:31,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:31,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:31,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:31,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:31,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:31,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:31,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:31,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:31,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:31,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:31,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:31,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:31,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:31,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:31,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:31,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:31,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:31,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:31,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:31,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:31,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:31,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:31,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:31,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:31,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:31,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:31,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:31,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:31,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:31,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:31,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:31,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:31,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:31,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:31,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:31,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:31,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:31,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:31,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:31,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:31,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:31,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:31,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:31,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:31,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:31,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:31,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:31,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:31,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:31,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:31,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:31,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:31,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:31,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:31,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:31,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:31,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:31,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:31,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:31,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:31,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:31,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:31,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:31,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:31,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:31,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:31,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:31,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:31,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:31,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:31,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:31,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:31,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:31,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:31,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:31,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:31,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:31,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:31,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:31,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:31,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:31,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:31,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:31,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:31,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:31,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:31,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:31,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:31,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:31,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:31,126 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:31,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59614 deadline: 1732378531114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:31,127 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:31,127 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:31,127 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:14:31,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:31,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:31,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:31,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:31,132 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:31,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59586 deadline: 1732378531121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:31,133 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:31,133 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:31,133 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:31,133 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:14:31,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59628 deadline: 1732378531122, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:31,134 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:31,134 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:31,134 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:14:31,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:31,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:31,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:31,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:31,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:31,138 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:31,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59638 deadline: 1732378531127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:31,139 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:31,139 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:31,139 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:14:31,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-23T16:14:31,204 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-23T16:14:31,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853. 2024-11-23T16:14:31,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853. as already flushing 2024-11-23T16:14:31,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853. 2024-11-23T16:14:31,205 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => cf5d4648fde06cb520ace381e0be9853, NAME => 'TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:14:31,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => cf5d4648fde06cb520ace381e0be9853, NAME => 'TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:14:31,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cf5d4648fde06cb520ace381e0be9853, NAME => 'TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cf5d4648fde06cb520ace381e0be9853, NAME => 'TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:14:31,241 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:31,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59614 deadline: 1732378531238, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:31,242 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:31,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59628 deadline: 1732378531238, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:31,242 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:31,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59586 deadline: 1732378531238, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:31,242 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:31,242 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:31,242 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:31,242 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:31,242 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:14:31,242 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:14:31,243 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:31,243 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:31,243 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:14:31,252 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:31,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59638 deadline: 1732378531247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:31,253 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:31,253 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:31,253 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:14:31,359 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-23T16:14:31,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853. 2024-11-23T16:14:31,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853. as already flushing 2024-11-23T16:14:31,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853. 2024-11-23T16:14:31,359 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => cf5d4648fde06cb520ace381e0be9853, NAME => 'TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:14:31,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => cf5d4648fde06cb520ace381e0be9853, NAME => 'TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:14:31,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cf5d4648fde06cb520ace381e0be9853, NAME => 'TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cf5d4648fde06cb520ace381e0be9853, NAME => 'TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:14:31,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-23T16:14:31,452 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:31,456 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:31,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59628 deadline: 1732378531448, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:31,456 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:31,456 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:31,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59614 deadline: 1732378531448, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:31,457 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:31,457 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:14:31,457 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:31,457 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:31,457 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:14:31,457 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:31,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59586 deadline: 1732378531452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:31,458 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:31,458 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:31,458 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:14:31,459 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112309de6b870b6c4a11b02106a02e0c0ae8_cf5d4648fde06cb520ace381e0be9853 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112309de6b870b6c4a11b02106a02e0c0ae8_cf5d4648fde06cb520ace381e0be9853 2024-11-23T16:14:31,461 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/.tmp/A/7336d857221a4c73bef299d957eca53c, store: [table=TestAcidGuarantees family=A region=cf5d4648fde06cb520ace381e0be9853] 2024-11-23T16:14:31,462 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/.tmp/A/7336d857221a4c73bef299d957eca53c is 175, key is test_row_1/A:col10/1732378471012/Put/seqid=0 2024-11-23T16:14:31,462 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:31,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59638 deadline: 1732378531458, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:31,463 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:31,463 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:31,463 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:14:31,467 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742045_1221 (size=31251) 2024-11-23T16:14:31,513 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-23T16:14:31,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853. 2024-11-23T16:14:31,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853. as already flushing 2024-11-23T16:14:31,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853. 2024-11-23T16:14:31,513 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => cf5d4648fde06cb520ace381e0be9853, NAME => 'TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:14:31,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => cf5d4648fde06cb520ace381e0be9853, NAME => 'TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:14:31,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cf5d4648fde06cb520ace381e0be9853, NAME => 'TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cf5d4648fde06cb520ace381e0be9853, NAME => 'TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:14:31,667 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-23T16:14:31,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853. 2024-11-23T16:14:31,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853. as already flushing 2024-11-23T16:14:31,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853. 2024-11-23T16:14:31,667 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => cf5d4648fde06cb520ace381e0be9853, NAME => 'TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:14:31,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => cf5d4648fde06cb520ace381e0be9853, NAME => 'TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:14:31,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cf5d4648fde06cb520ace381e0be9853, NAME => 'TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cf5d4648fde06cb520ace381e0be9853, NAME => 'TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:14:31,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-23T16:14:31,770 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:31,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59614 deadline: 1732378531768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:31,771 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:31,771 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:31,771 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:14:31,771 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:31,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59628 deadline: 1732378531768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:31,771 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:31,771 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:31,771 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:14:31,771 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:31,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59586 deadline: 1732378531769, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:31,772 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:31,772 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:31,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59638 deadline: 1732378531769, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:31,772 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:31,772 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:14:31,773 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:31,773 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:31,773 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:14:31,820 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-23T16:14:31,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853. 2024-11-23T16:14:31,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853. as already flushing 2024-11-23T16:14:31,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853. 2024-11-23T16:14:31,820 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => cf5d4648fde06cb520ace381e0be9853, NAME => 'TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:14:31,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => cf5d4648fde06cb520ace381e0be9853, NAME => 'TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:14:31,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cf5d4648fde06cb520ace381e0be9853, NAME => 'TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cf5d4648fde06cb520ace381e0be9853, NAME => 'TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:14:31,868 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=373, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/.tmp/A/7336d857221a4c73bef299d957eca53c 2024-11-23T16:14:31,878 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/.tmp/B/33a0fe6442754c2d96c63a1878c016a1 is 50, key is test_row_1/B:col10/1732378471012/Put/seqid=0 2024-11-23T16:14:31,906 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742046_1222 (size=9857) 2024-11-23T16:14:31,974 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-23T16:14:31,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853. 2024-11-23T16:14:31,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853. as already flushing 2024-11-23T16:14:31,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853. 2024-11-23T16:14:31,974 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => cf5d4648fde06cb520ace381e0be9853, NAME => 'TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:14:31,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => cf5d4648fde06cb520ace381e0be9853, NAME => 'TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:14:31,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cf5d4648fde06cb520ace381e0be9853, NAME => 'TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cf5d4648fde06cb520ace381e0be9853, NAME => 'TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:14:32,127 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-23T16:14:32,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853. 2024-11-23T16:14:32,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853. as already flushing 2024-11-23T16:14:32,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853. 2024-11-23T16:14:32,127 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => cf5d4648fde06cb520ace381e0be9853, NAME => 'TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:14:32,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => cf5d4648fde06cb520ace381e0be9853, NAME => 'TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:14:32,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cf5d4648fde06cb520ace381e0be9853, NAME => 'TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cf5d4648fde06cb520ace381e0be9853, NAME => 'TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:14:32,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-23T16:14:32,280 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:32,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59614 deadline: 1732378532278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:32,280 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:32,280 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:32,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59638 deadline: 1732378532278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:32,280 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:32,280 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:14:32,280 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-23T16:14:32,281 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:32,281 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:32,281 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:32,281 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:14:32,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59628 deadline: 1732378532278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:32,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853. 2024-11-23T16:14:32,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853. as already flushing 2024-11-23T16:14:32,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853. 2024-11-23T16:14:32,281 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:32,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59586 deadline: 1732378532279, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:32,281 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => cf5d4648fde06cb520ace381e0be9853, NAME => 'TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:14:32,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => cf5d4648fde06cb520ace381e0be9853, NAME => 'TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:14:32,282 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:32,282 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:32,282 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:14:32,282 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:32,282 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:32,282 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:14:32,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cf5d4648fde06cb520ace381e0be9853, NAME => 'TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cf5d4648fde06cb520ace381e0be9853, NAME => 'TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:14:32,307 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=373 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/.tmp/B/33a0fe6442754c2d96c63a1878c016a1 2024-11-23T16:14:32,326 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/.tmp/C/255627e4041f4998861702bde7d081fa is 50, key is test_row_1/C:col10/1732378471012/Put/seqid=0 2024-11-23T16:14:32,362 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742047_1223 (size=9857) 2024-11-23T16:14:32,363 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=373 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/.tmp/C/255627e4041f4998861702bde7d081fa 2024-11-23T16:14:32,371 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/.tmp/A/7336d857221a4c73bef299d957eca53c as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/A/7336d857221a4c73bef299d957eca53c 2024-11-23T16:14:32,376 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/A/7336d857221a4c73bef299d957eca53c, entries=150, sequenceid=373, filesize=30.5 K 2024-11-23T16:14:32,377 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/.tmp/B/33a0fe6442754c2d96c63a1878c016a1 as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/B/33a0fe6442754c2d96c63a1878c016a1 2024-11-23T16:14:32,383 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/B/33a0fe6442754c2d96c63a1878c016a1, entries=100, sequenceid=373, filesize=9.6 K 2024-11-23T16:14:32,386 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/.tmp/C/255627e4041f4998861702bde7d081fa as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/C/255627e4041f4998861702bde7d081fa 2024-11-23T16:14:32,393 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/C/255627e4041f4998861702bde7d081fa, entries=100, sequenceid=373, filesize=9.6 K 2024-11-23T16:14:32,394 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for cf5d4648fde06cb520ace381e0be9853 in 1381ms, sequenceid=373, compaction requested=true 2024-11-23T16:14:32,394 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cf5d4648fde06cb520ace381e0be9853: 2024-11-23T16:14:32,394 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cf5d4648fde06cb520ace381e0be9853:A, priority=-2147483648, current under compaction store size is 1 2024-11-23T16:14:32,394 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T16:14:32,394 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cf5d4648fde06cb520ace381e0be9853:B, priority=-2147483648, current under compaction store size is 2 2024-11-23T16:14:32,394 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T16:14:32,394 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cf5d4648fde06cb520ace381e0be9853:C, priority=-2147483648, current under compaction store size is 3 2024-11-23T16:14:32,394 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T16:14:32,394 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T16:14:32,394 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T16:14:32,395 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35209 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T16:14:32,395 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HStore(1541): cf5d4648fde06cb520ace381e0be9853/B is initiating minor compaction (all files) 2024-11-23T16:14:32,395 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 111895 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T16:14:32,395 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cf5d4648fde06cb520ace381e0be9853/B in TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853. 2024-11-23T16:14:32,395 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HStore(1541): cf5d4648fde06cb520ace381e0be9853/A is initiating minor compaction (all files) 2024-11-23T16:14:32,395 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cf5d4648fde06cb520ace381e0be9853/A in TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853. 2024-11-23T16:14:32,395 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/B/01c7c21da7c244dfaa73f9591258d0be, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/B/9b7fef690e4449549e61a0f00cca7b9f, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/B/33a0fe6442754c2d96c63a1878c016a1] into tmpdir=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/.tmp, totalSize=34.4 K 2024-11-23T16:14:32,395 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/A/2bdf1def87fc4ffca6fac4d59284cd8f, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/A/0e66cbdcb82d41e4ae53bd3397947e6c, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/A/7336d857221a4c73bef299d957eca53c] into tmpdir=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/.tmp, totalSize=109.3 K 2024-11-23T16:14:32,396 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853. 2024-11-23T16:14:32,396 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853. files: [hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/A/2bdf1def87fc4ffca6fac4d59284cd8f, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/A/0e66cbdcb82d41e4ae53bd3397947e6c, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/A/7336d857221a4c73bef299d957eca53c] 2024-11-23T16:14:32,396 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.Compactor(225): Compacting 01c7c21da7c244dfaa73f9591258d0be, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=337, earliestPutTs=1732378468588 2024-11-23T16:14:32,396 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.Compactor(225): Compacting 9b7fef690e4449549e61a0f00cca7b9f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=360, earliestPutTs=1732378469278 2024-11-23T16:14:32,397 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.Compactor(225): Compacting 33a0fe6442754c2d96c63a1878c016a1, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=373, earliestPutTs=1732378470451 2024-11-23T16:14:32,397 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2bdf1def87fc4ffca6fac4d59284cd8f, keycount=150, bloomtype=ROW, size=31.3 K, encoding=NONE, compression=NONE, seqNum=337, earliestPutTs=1732378468588 2024-11-23T16:14:32,397 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0e66cbdcb82d41e4ae53bd3397947e6c, keycount=250, bloomtype=ROW, size=47.5 K, encoding=NONE, compression=NONE, seqNum=360, earliestPutTs=1732378469278 2024-11-23T16:14:32,398 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7336d857221a4c73bef299d957eca53c, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=373, earliestPutTs=1732378470445 2024-11-23T16:14:32,410 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=cf5d4648fde06cb520ace381e0be9853] 2024-11-23T16:14:32,411 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cf5d4648fde06cb520ace381e0be9853#B#compaction#183 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T16:14:32,411 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/.tmp/B/02563ea2ab644c9090436672bee3193d is 50, key is test_row_0/B:col10/1732378469284/Put/seqid=0 2024-11-23T16:14:32,421 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241123a0d599bdf2f642db8f9b4af66bb05c3c_cf5d4648fde06cb520ace381e0be9853 store=[table=TestAcidGuarantees family=A region=cf5d4648fde06cb520ace381e0be9853] 2024-11-23T16:14:32,423 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241123a0d599bdf2f642db8f9b4af66bb05c3c_cf5d4648fde06cb520ace381e0be9853, store=[table=TestAcidGuarantees family=A region=cf5d4648fde06cb520ace381e0be9853] 2024-11-23T16:14:32,423 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241123a0d599bdf2f642db8f9b4af66bb05c3c_cf5d4648fde06cb520ace381e0be9853 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=cf5d4648fde06cb520ace381e0be9853] 2024-11-23T16:14:32,430 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742048_1224 (size=13153) 2024-11-23T16:14:32,434 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742049_1225 (size=4469) 2024-11-23T16:14:32,435 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-23T16:14:32,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853. 2024-11-23T16:14:32,435 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2902): Flushing cf5d4648fde06cb520ace381e0be9853 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-23T16:14:32,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf5d4648fde06cb520ace381e0be9853, store=A 2024-11-23T16:14:32,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-23T16:14:32,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf5d4648fde06cb520ace381e0be9853, store=B 2024-11-23T16:14:32,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-23T16:14:32,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf5d4648fde06cb520ace381e0be9853, store=C 2024-11-23T16:14:32,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-23T16:14:32,436 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/.tmp/B/02563ea2ab644c9090436672bee3193d as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/B/02563ea2ab644c9090436672bee3193d 2024-11-23T16:14:32,446 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cf5d4648fde06cb520ace381e0be9853/B of cf5d4648fde06cb520ace381e0be9853 into 02563ea2ab644c9090436672bee3193d(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T16:14:32,446 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cf5d4648fde06cb520ace381e0be9853: 2024-11-23T16:14:32,446 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., storeName=cf5d4648fde06cb520ace381e0be9853/B, priority=13, startTime=1732378472394; duration=0sec 2024-11-23T16:14:32,447 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T16:14:32,447 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cf5d4648fde06cb520ace381e0be9853:B 2024-11-23T16:14:32,447 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T16:14:32,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411235b2ab43559314eafb9ebf2d7b1b45286_cf5d4648fde06cb520ace381e0be9853 is 50, key is test_row_0/A:col10/1732378471122/Put/seqid=0 2024-11-23T16:14:32,449 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35209 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T16:14:32,449 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HStore(1541): cf5d4648fde06cb520ace381e0be9853/C is initiating minor compaction (all files) 2024-11-23T16:14:32,449 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cf5d4648fde06cb520ace381e0be9853/C in TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853. 2024-11-23T16:14:32,449 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/C/84dc627c32b948c9aaf05a80aa818018, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/C/a74a14f34a3542adb828cfefbebf05d9, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/C/255627e4041f4998861702bde7d081fa] into tmpdir=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/.tmp, totalSize=34.4 K 2024-11-23T16:14:32,450 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.Compactor(225): Compacting 84dc627c32b948c9aaf05a80aa818018, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=337, earliestPutTs=1732378468588 2024-11-23T16:14:32,450 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.Compactor(225): Compacting a74a14f34a3542adb828cfefbebf05d9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=360, earliestPutTs=1732378469278 2024-11-23T16:14:32,450 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.Compactor(225): Compacting 255627e4041f4998861702bde7d081fa, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=373, earliestPutTs=1732378470451 2024-11-23T16:14:32,456 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742050_1226 (size=12454) 2024-11-23T16:14:32,467 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cf5d4648fde06cb520ace381e0be9853#C#compaction#186 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-23T16:14:32,468 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/.tmp/C/65ed4ba4b704414f8ea6e20091576b02 is 50, key is test_row_0/C:col10/1732378469284/Put/seqid=0 2024-11-23T16:14:32,482 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742051_1227 (size=13153) 2024-11-23T16:14:32,490 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/.tmp/C/65ed4ba4b704414f8ea6e20091576b02 as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/C/65ed4ba4b704414f8ea6e20091576b02 2024-11-23T16:14:32,496 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cf5d4648fde06cb520ace381e0be9853/C of cf5d4648fde06cb520ace381e0be9853 into 65ed4ba4b704414f8ea6e20091576b02(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T16:14:32,496 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cf5d4648fde06cb520ace381e0be9853: 2024-11-23T16:14:32,496 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., storeName=cf5d4648fde06cb520ace381e0be9853/C, priority=13, startTime=1732378472394; duration=0sec 2024-11-23T16:14:32,496 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T16:14:32,497 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cf5d4648fde06cb520ace381e0be9853:C 2024-11-23T16:14:32,836 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): cf5d4648fde06cb520ace381e0be9853#A#compaction#184 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T16:14:32,837 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/.tmp/A/6859e4c03a244fec843582e0deaf9d37 is 175, key is test_row_0/A:col10/1732378469284/Put/seqid=0 2024-11-23T16:14:32,852 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742052_1228 (size=32214) 2024-11-23T16:14:32,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:32,862 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411235b2ab43559314eafb9ebf2d7b1b45286_cf5d4648fde06cb520ace381e0be9853 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411235b2ab43559314eafb9ebf2d7b1b45286_cf5d4648fde06cb520ace381e0be9853 2024-11-23T16:14:32,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/.tmp/A/b06c9f4c58304a0d8dd2f12d433782fb, store: [table=TestAcidGuarantees family=A region=cf5d4648fde06cb520ace381e0be9853] 2024-11-23T16:14:32,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/.tmp/A/b06c9f4c58304a0d8dd2f12d433782fb is 175, key is test_row_0/A:col10/1732378471122/Put/seqid=0 2024-11-23T16:14:32,873 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742053_1229 (size=31255) 2024-11-23T16:14:33,073 INFO [Thread-637 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-637. 2024-11-23T16:14:33,073 DEBUG [Thread-637 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-23T16:14:33,073 DEBUG [Thread-637 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T16:14:33,073 DEBUG [Thread-637 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T16:14:33,073 INFO [Thread-635 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-635. 2024-11-23T16:14:33,073 DEBUG [Thread-635 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-23T16:14:33,074 DEBUG [Thread-635 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T16:14:33,074 DEBUG [Thread-635 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T16:14:33,074 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-23T16:14:33,074 INFO [Thread-633 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-633. 2024-11-23T16:14:33,074 DEBUG [Thread-633 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-23T16:14:33,074 DEBUG [Thread-633 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T16:14:33,074 DEBUG [Thread-633 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T16:14:33,074 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-23T16:14:33,075 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-23T16:14:33,080 INFO [Thread-639 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-639. 2024-11-23T16:14:33,080 DEBUG [Thread-639 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-23T16:14:33,080 DEBUG [Thread-639 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T16:14:33,080 DEBUG [Thread-639 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T16:14:33,080 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-23T16:14:33,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-23T16:14:33,259 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/.tmp/A/6859e4c03a244fec843582e0deaf9d37 as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/A/6859e4c03a244fec843582e0deaf9d37 2024-11-23T16:14:33,266 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cf5d4648fde06cb520ace381e0be9853/A of cf5d4648fde06cb520ace381e0be9853 into 6859e4c03a244fec843582e0deaf9d37(size=31.5 K), total size for store is 31.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T16:14:33,266 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cf5d4648fde06cb520ace381e0be9853: 2024-11-23T16:14:33,266 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., storeName=cf5d4648fde06cb520ace381e0be9853/A, priority=13, startTime=1732378472394; duration=0sec 2024-11-23T16:14:33,266 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T16:14:33,266 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cf5d4648fde06cb520ace381e0be9853:A 2024-11-23T16:14:33,274 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=393, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/.tmp/A/b06c9f4c58304a0d8dd2f12d433782fb 2024-11-23T16:14:33,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/.tmp/B/630b68fad74a49db8e1c70dde5222cb4 is 50, key is test_row_0/B:col10/1732378471122/Put/seqid=0 2024-11-23T16:14:33,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(8855): Flush requested on cf5d4648fde06cb520ace381e0be9853 2024-11-23T16:14:33,290 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853. as already flushing 2024-11-23T16:14:33,290 INFO [Thread-624 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-624. 2024-11-23T16:14:33,290 DEBUG [Thread-624 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-23T16:14:33,290 DEBUG [Thread-624 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T16:14:33,290 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742054_1230 (size=12301) 2024-11-23T16:14:33,291 DEBUG [Thread-624 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T16:14:33,291 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-23T16:14:33,291 INFO [Thread-626 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-626. 2024-11-23T16:14:33,291 DEBUG [Thread-626 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-23T16:14:33,291 DEBUG [Thread-626 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T16:14:33,291 DEBUG [Thread-626 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T16:14:33,291 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-23T16:14:33,301 INFO [Thread-622 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-622. 2024-11-23T16:14:33,301 INFO [Thread-630 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-630. 2024-11-23T16:14:33,301 DEBUG [Thread-630 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-23T16:14:33,301 DEBUG [Thread-622 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-23T16:14:33,301 DEBUG [Thread-630 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T16:14:33,301 DEBUG [Thread-622 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T16:14:33,301 DEBUG [Thread-630 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T16:14:33,301 DEBUG [Thread-622 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T16:14:33,301 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-23T16:14:33,301 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-23T16:14:33,691 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=393 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/.tmp/B/630b68fad74a49db8e1c70dde5222cb4 2024-11-23T16:14:33,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/.tmp/C/c7579036ccf84bf5ad383446f8c84c89 is 50, key is test_row_0/C:col10/1732378471122/Put/seqid=0 2024-11-23T16:14:33,703 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742055_1231 (size=12301) 2024-11-23T16:14:34,104 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=393 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/.tmp/C/c7579036ccf84bf5ad383446f8c84c89 2024-11-23T16:14:34,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/.tmp/A/b06c9f4c58304a0d8dd2f12d433782fb as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/A/b06c9f4c58304a0d8dd2f12d433782fb 2024-11-23T16:14:34,119 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/A/b06c9f4c58304a0d8dd2f12d433782fb, entries=150, sequenceid=393, filesize=30.5 K 2024-11-23T16:14:34,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/.tmp/B/630b68fad74a49db8e1c70dde5222cb4 as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/B/630b68fad74a49db8e1c70dde5222cb4 2024-11-23T16:14:34,140 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/B/630b68fad74a49db8e1c70dde5222cb4, entries=150, sequenceid=393, filesize=12.0 K 2024-11-23T16:14:34,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/.tmp/C/c7579036ccf84bf5ad383446f8c84c89 as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/C/c7579036ccf84bf5ad383446f8c84c89 2024-11-23T16:14:34,146 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/C/c7579036ccf84bf5ad383446f8c84c89, entries=150, sequenceid=393, filesize=12.0 K 2024-11-23T16:14:34,147 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=26.84 KB/27480 for cf5d4648fde06cb520ace381e0be9853 in 1712ms, sequenceid=393, compaction requested=false 2024-11-23T16:14:34,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2603): Flush status journal for cf5d4648fde06cb520ace381e0be9853: 2024-11-23T16:14:34,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853. 2024-11-23T16:14:34,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=53 2024-11-23T16:14:34,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.HMaster(4169): Remote procedure done, pid=53 2024-11-23T16:14:34,151 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=53, resume processing ppid=52 2024-11-23T16:14:34,151 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=53, ppid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.0960 sec 2024-11-23T16:14:34,153 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees in 3.1020 sec 2024-11-23T16:14:35,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-23T16:14:35,209 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-23T16:14:36,839 INFO [Thread-628 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-628. 2024-11-23T16:14:36,839 DEBUG [Thread-628 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-23T16:14:36,839 DEBUG [Thread-628 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T16:14:36,839 DEBUG [Thread-628 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T16:14:36,839 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-23T16:14:36,840 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-23T16:14:36,840 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 49 2024-11-23T16:14:36,840 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 80 2024-11-23T16:14:36,840 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 75 2024-11-23T16:14:36,840 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 27 2024-11-23T16:14:36,840 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 85 2024-11-23T16:14:36,840 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-23T16:14:36,840 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2739 2024-11-23T16:14:36,840 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2521 2024-11-23T16:14:36,840 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-23T16:14:36,840 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1140 2024-11-23T16:14:36,840 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3419 rows 2024-11-23T16:14:36,840 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1147 2024-11-23T16:14:36,840 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3440 rows 2024-11-23T16:14:36,840 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-23T16:14:36,840 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobMixedAtomicity(AcidGuaranteesTestBase.java:132) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-23T16:14:36,840 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T16:14:36,840 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T16:14:36,840 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-23T16:14:36,841 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-23T16:14:36,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] procedure2.ProcedureExecutor(1139): Stored pid=54, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-23T16:14:36,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-23T16:14:36,845 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732378476844"}]},"ts":"1732378476844"} 2024-11-23T16:14:36,846 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-23T16:14:36,846 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-23T16:14:36,847 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=55, ppid=54, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-23T16:14:36,849 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=56, ppid=55, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=cf5d4648fde06cb520ace381e0be9853, UNASSIGN}] 2024-11-23T16:14:36,849 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=56, ppid=55, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=cf5d4648fde06cb520ace381e0be9853, UNASSIGN 2024-11-23T16:14:36,850 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=56 updating hbase:meta row=cf5d4648fde06cb520ace381e0be9853, regionState=CLOSING, regionLocation=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:36,851 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=56, ppid=55, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=cf5d4648fde06cb520ace381e0be9853, UNASSIGN because future has completed 2024-11-23T16:14:36,852 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-23T16:14:36,852 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=57, ppid=56, state=RUNNABLE, hasLock=false; CloseRegionProcedure cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375}] 2024-11-23T16:14:36,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-23T16:14:37,005 INFO [RS_CLOSE_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] handler.UnassignRegionHandler(122): Close cf5d4648fde06cb520ace381e0be9853 2024-11-23T16:14:37,005 DEBUG [RS_CLOSE_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-23T16:14:37,005 DEBUG [RS_CLOSE_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(1722): Closing cf5d4648fde06cb520ace381e0be9853, disabling compactions & flushes 2024-11-23T16:14:37,005 INFO [RS_CLOSE_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853. 2024-11-23T16:14:37,005 DEBUG [RS_CLOSE_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853. 2024-11-23T16:14:37,005 DEBUG [RS_CLOSE_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853. after waiting 0 ms 2024-11-23T16:14:37,005 DEBUG [RS_CLOSE_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853. 2024-11-23T16:14:37,005 INFO [RS_CLOSE_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(2902): Flushing cf5d4648fde06cb520ace381e0be9853 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-23T16:14:37,006 DEBUG [RS_CLOSE_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf5d4648fde06cb520ace381e0be9853, store=A 2024-11-23T16:14:37,006 DEBUG [RS_CLOSE_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-23T16:14:37,006 DEBUG [RS_CLOSE_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf5d4648fde06cb520ace381e0be9853, store=B 2024-11-23T16:14:37,006 DEBUG [RS_CLOSE_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-23T16:14:37,006 DEBUG [RS_CLOSE_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf5d4648fde06cb520ace381e0be9853, store=C 2024-11-23T16:14:37,006 DEBUG [RS_CLOSE_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-23T16:14:37,015 DEBUG [RS_CLOSE_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241123e48a7e6d27ae41cf9bae65a2fb3db1a5_cf5d4648fde06cb520ace381e0be9853 is 50, key is test_row_0/A:col10/1732378473288/Put/seqid=0 2024-11-23T16:14:37,031 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742056_1232 (size=12454) 2024-11-23T16:14:37,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-23T16:14:37,433 DEBUG [RS_CLOSE_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:37,459 INFO [RS_CLOSE_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241123e48a7e6d27ae41cf9bae65a2fb3db1a5_cf5d4648fde06cb520ace381e0be9853 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123e48a7e6d27ae41cf9bae65a2fb3db1a5_cf5d4648fde06cb520ace381e0be9853 2024-11-23T16:14:37,460 DEBUG [RS_CLOSE_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/.tmp/A/ef493ade9f9b4566b0592f39947a1276, store: [table=TestAcidGuarantees family=A region=cf5d4648fde06cb520ace381e0be9853] 2024-11-23T16:14:37,461 DEBUG [RS_CLOSE_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/.tmp/A/ef493ade9f9b4566b0592f39947a1276 is 175, key is test_row_0/A:col10/1732378473288/Put/seqid=0 2024-11-23T16:14:37,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-23T16:14:37,470 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742057_1233 (size=31255) 2024-11-23T16:14:37,474 INFO [RS_CLOSE_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=404, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/.tmp/A/ef493ade9f9b4566b0592f39947a1276 2024-11-23T16:14:37,482 DEBUG [RS_CLOSE_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/.tmp/B/ea3a4059fb48448aac7da181f4d01bcb is 50, key is test_row_0/B:col10/1732378473288/Put/seqid=0 2024-11-23T16:14:37,488 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742058_1234 (size=12301) 2024-11-23T16:14:37,491 INFO [RS_CLOSE_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=404 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/.tmp/B/ea3a4059fb48448aac7da181f4d01bcb 2024-11-23T16:14:37,507 DEBUG [RS_CLOSE_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/.tmp/C/65711d740480460bb754bdaa626542ba is 50, key is test_row_0/C:col10/1732378473288/Put/seqid=0 2024-11-23T16:14:37,514 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742059_1235 (size=12301) 2024-11-23T16:14:37,915 INFO [RS_CLOSE_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=404 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/.tmp/C/65711d740480460bb754bdaa626542ba 2024-11-23T16:14:37,920 DEBUG [RS_CLOSE_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/.tmp/A/ef493ade9f9b4566b0592f39947a1276 as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/A/ef493ade9f9b4566b0592f39947a1276 2024-11-23T16:14:37,926 INFO [RS_CLOSE_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/A/ef493ade9f9b4566b0592f39947a1276, entries=150, sequenceid=404, filesize=30.5 K 2024-11-23T16:14:37,927 DEBUG [RS_CLOSE_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/.tmp/B/ea3a4059fb48448aac7da181f4d01bcb as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/B/ea3a4059fb48448aac7da181f4d01bcb 2024-11-23T16:14:37,932 INFO [RS_CLOSE_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/B/ea3a4059fb48448aac7da181f4d01bcb, entries=150, sequenceid=404, filesize=12.0 K 2024-11-23T16:14:37,933 DEBUG [RS_CLOSE_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/.tmp/C/65711d740480460bb754bdaa626542ba as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/C/65711d740480460bb754bdaa626542ba 2024-11-23T16:14:37,937 INFO [RS_CLOSE_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/C/65711d740480460bb754bdaa626542ba, entries=150, sequenceid=404, filesize=12.0 K 2024-11-23T16:14:37,938 INFO [RS_CLOSE_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for cf5d4648fde06cb520ace381e0be9853 in 933ms, sequenceid=404, compaction requested=true 2024-11-23T16:14:37,939 DEBUG [StoreCloser-TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/A/389baa2344bd4e88a84af99676683d7c, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/A/a8c0de2db7aa45a8b430bae628f15ee0, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/A/fd413a345eeb47718b7fd8958867c552, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/A/93c3613f1946487e90376f2ac7ba6b5b, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/A/dc257c88d995485a824a51457199c7f1, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/A/1a55f4f0a8e84edaba42163864b60e59, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/A/965712ddc1ff49dfb4706f98e3226a4b, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/A/90db0d62c1304f1f92806b95265a9106, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/A/fb2a54a1f0c640ea8f92799b615c30e9, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/A/274abfa78d934cd49d56bc1cc3ac58fd, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/A/0aa01d4258f448abbb96bb919585b2ba, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/A/040a9bb41af84f1dbe6a0145ef6dd3d2, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/A/fbde40db105a4a57a29f1abe87a38595, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/A/63aca72b8ed64727b1f106e1ae4552b3, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/A/5477215922a748ef8f1ee5c942bf9e3f, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/A/e53a403ff9154cc29a7d893952dbc50b, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/A/547aea4f4f6e4109818f11a11d224b63, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/A/24f9ae4954a445b4804cc0064321f70a, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/A/c19e36f81ddb44a9a83b553f1dcbc18e, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/A/1d08693d212c422786dfeceb72435999, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/A/a3966beaa6f349b88e01530018370935, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/A/0f7dc6b491da459bb539c172c9e15d81, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/A/2bdf1def87fc4ffca6fac4d59284cd8f, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/A/0e66cbdcb82d41e4ae53bd3397947e6c, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/A/7336d857221a4c73bef299d957eca53c] to archive 2024-11-23T16:14:37,940 DEBUG [StoreCloser-TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-23T16:14:37,943 DEBUG [StoreCloser-TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/A/389baa2344bd4e88a84af99676683d7c to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/A/389baa2344bd4e88a84af99676683d7c 2024-11-23T16:14:37,944 DEBUG [StoreCloser-TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/A/a8c0de2db7aa45a8b430bae628f15ee0 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/A/a8c0de2db7aa45a8b430bae628f15ee0 2024-11-23T16:14:37,946 DEBUG [StoreCloser-TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/A/fd413a345eeb47718b7fd8958867c552 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/A/fd413a345eeb47718b7fd8958867c552 2024-11-23T16:14:37,947 DEBUG [StoreCloser-TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/A/93c3613f1946487e90376f2ac7ba6b5b to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/A/93c3613f1946487e90376f2ac7ba6b5b 2024-11-23T16:14:37,948 DEBUG [StoreCloser-TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/A/dc257c88d995485a824a51457199c7f1 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/A/dc257c88d995485a824a51457199c7f1 2024-11-23T16:14:37,949 DEBUG [StoreCloser-TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/A/1a55f4f0a8e84edaba42163864b60e59 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/A/1a55f4f0a8e84edaba42163864b60e59 2024-11-23T16:14:37,950 DEBUG [StoreCloser-TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/A/965712ddc1ff49dfb4706f98e3226a4b to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/A/965712ddc1ff49dfb4706f98e3226a4b 2024-11-23T16:14:37,951 DEBUG [StoreCloser-TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/A/90db0d62c1304f1f92806b95265a9106 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/A/90db0d62c1304f1f92806b95265a9106 2024-11-23T16:14:37,952 DEBUG [StoreCloser-TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/A/fb2a54a1f0c640ea8f92799b615c30e9 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/A/fb2a54a1f0c640ea8f92799b615c30e9 2024-11-23T16:14:37,953 DEBUG [StoreCloser-TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/A/274abfa78d934cd49d56bc1cc3ac58fd to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/A/274abfa78d934cd49d56bc1cc3ac58fd 2024-11-23T16:14:37,954 DEBUG [StoreCloser-TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/A/0aa01d4258f448abbb96bb919585b2ba to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/A/0aa01d4258f448abbb96bb919585b2ba 2024-11-23T16:14:37,955 DEBUG [StoreCloser-TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/A/040a9bb41af84f1dbe6a0145ef6dd3d2 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/A/040a9bb41af84f1dbe6a0145ef6dd3d2 2024-11-23T16:14:37,956 DEBUG [StoreCloser-TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/A/fbde40db105a4a57a29f1abe87a38595 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/A/fbde40db105a4a57a29f1abe87a38595 2024-11-23T16:14:37,957 DEBUG [StoreCloser-TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/A/63aca72b8ed64727b1f106e1ae4552b3 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/A/63aca72b8ed64727b1f106e1ae4552b3 2024-11-23T16:14:37,958 DEBUG [StoreCloser-TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/A/5477215922a748ef8f1ee5c942bf9e3f to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/A/5477215922a748ef8f1ee5c942bf9e3f 2024-11-23T16:14:37,959 DEBUG [StoreCloser-TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/A/e53a403ff9154cc29a7d893952dbc50b to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/A/e53a403ff9154cc29a7d893952dbc50b 2024-11-23T16:14:37,960 DEBUG [StoreCloser-TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/A/547aea4f4f6e4109818f11a11d224b63 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/A/547aea4f4f6e4109818f11a11d224b63 2024-11-23T16:14:37,961 DEBUG [StoreCloser-TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/A/24f9ae4954a445b4804cc0064321f70a to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/A/24f9ae4954a445b4804cc0064321f70a 2024-11-23T16:14:37,962 DEBUG [StoreCloser-TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/A/c19e36f81ddb44a9a83b553f1dcbc18e to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/A/c19e36f81ddb44a9a83b553f1dcbc18e 2024-11-23T16:14:37,963 DEBUG [StoreCloser-TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/A/1d08693d212c422786dfeceb72435999 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/A/1d08693d212c422786dfeceb72435999 2024-11-23T16:14:37,964 DEBUG [StoreCloser-TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/A/a3966beaa6f349b88e01530018370935 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/A/a3966beaa6f349b88e01530018370935 2024-11-23T16:14:37,965 DEBUG [StoreCloser-TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/A/0f7dc6b491da459bb539c172c9e15d81 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/A/0f7dc6b491da459bb539c172c9e15d81 2024-11-23T16:14:37,966 DEBUG [StoreCloser-TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/A/2bdf1def87fc4ffca6fac4d59284cd8f to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/A/2bdf1def87fc4ffca6fac4d59284cd8f 2024-11-23T16:14:37,967 DEBUG [StoreCloser-TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/A/0e66cbdcb82d41e4ae53bd3397947e6c to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/A/0e66cbdcb82d41e4ae53bd3397947e6c 2024-11-23T16:14:37,968 DEBUG [StoreCloser-TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/A/7336d857221a4c73bef299d957eca53c to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/A/7336d857221a4c73bef299d957eca53c 2024-11-23T16:14:37,969 DEBUG [StoreCloser-TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/B/a4ba58cfe0284efa8306d9f0f422a16c, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/B/0eaab5d5995441b5aa72f557fd1ea17d, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/B/69dda82214634ca8a905b57d1914de79, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/B/82d4234caa5043b3b53db13b01f44556, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/B/a04ca98c2c964164a6d8632875d0846d, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/B/1d598d0fa2594f62a6ff85d4131f2702, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/B/47365aaa2d7e4933b4b0be9429a13d88, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/B/d764ff88804e4a3f906461d54b49dddb, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/B/0718a41c649e459a8be955af356e718c, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/B/34f4da6344f04de99e859f831ed547ea, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/B/5cfcd5ab3a6249a7ac4da7a37cab896c, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/B/62abb0de1c19491e98b7b8869e639b0c, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/B/d6a4fa3d2a37434b9de8157cc23f73bf, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/B/c6d367246e934bdaa61003a704765ac1, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/B/787bc5f7e6e34bf7b0e06e9da46894c1, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/B/132c7e3c8f734abb862221695a55b037, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/B/e2a75731746a43719653ec1be70a2a2b, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/B/66abafe5050742da819745e24a2ae177, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/B/cb03383e6607400c8e097af59514a54a, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/B/bee71088d6ba4c8e99248cef4a1e1ca5, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/B/66dee4c6ddbf4593a032fca6ec7c2e6b, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/B/01c7c21da7c244dfaa73f9591258d0be, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/B/1174617488a94552afa9e1a3694e44a1, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/B/9b7fef690e4449549e61a0f00cca7b9f, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/B/33a0fe6442754c2d96c63a1878c016a1] to archive 2024-11-23T16:14:37,970 DEBUG [StoreCloser-TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-23T16:14:37,971 DEBUG [StoreCloser-TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/B/a4ba58cfe0284efa8306d9f0f422a16c to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/B/a4ba58cfe0284efa8306d9f0f422a16c 2024-11-23T16:14:37,972 DEBUG [StoreCloser-TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/B/0eaab5d5995441b5aa72f557fd1ea17d to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/B/0eaab5d5995441b5aa72f557fd1ea17d 2024-11-23T16:14:37,973 DEBUG [StoreCloser-TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/B/69dda82214634ca8a905b57d1914de79 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/B/69dda82214634ca8a905b57d1914de79 2024-11-23T16:14:37,974 DEBUG [StoreCloser-TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/B/82d4234caa5043b3b53db13b01f44556 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/B/82d4234caa5043b3b53db13b01f44556 2024-11-23T16:14:37,975 DEBUG [StoreCloser-TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/B/a04ca98c2c964164a6d8632875d0846d to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/B/a04ca98c2c964164a6d8632875d0846d 2024-11-23T16:14:37,976 DEBUG [StoreCloser-TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/B/1d598d0fa2594f62a6ff85d4131f2702 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/B/1d598d0fa2594f62a6ff85d4131f2702 2024-11-23T16:14:37,977 DEBUG [StoreCloser-TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/B/47365aaa2d7e4933b4b0be9429a13d88 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/B/47365aaa2d7e4933b4b0be9429a13d88 2024-11-23T16:14:37,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-23T16:14:37,978 DEBUG [StoreCloser-TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/B/d764ff88804e4a3f906461d54b49dddb to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/B/d764ff88804e4a3f906461d54b49dddb 2024-11-23T16:14:37,979 DEBUG [StoreCloser-TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/B/0718a41c649e459a8be955af356e718c to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/B/0718a41c649e459a8be955af356e718c 2024-11-23T16:14:37,980 DEBUG [StoreCloser-TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/B/34f4da6344f04de99e859f831ed547ea to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/B/34f4da6344f04de99e859f831ed547ea 2024-11-23T16:14:37,981 DEBUG [StoreCloser-TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/B/5cfcd5ab3a6249a7ac4da7a37cab896c to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/B/5cfcd5ab3a6249a7ac4da7a37cab896c 2024-11-23T16:14:37,982 DEBUG [StoreCloser-TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/B/62abb0de1c19491e98b7b8869e639b0c to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/B/62abb0de1c19491e98b7b8869e639b0c 2024-11-23T16:14:37,983 DEBUG [StoreCloser-TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/B/d6a4fa3d2a37434b9de8157cc23f73bf to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/B/d6a4fa3d2a37434b9de8157cc23f73bf 2024-11-23T16:14:37,984 DEBUG [StoreCloser-TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/B/c6d367246e934bdaa61003a704765ac1 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/B/c6d367246e934bdaa61003a704765ac1 2024-11-23T16:14:37,985 DEBUG [StoreCloser-TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/B/787bc5f7e6e34bf7b0e06e9da46894c1 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/B/787bc5f7e6e34bf7b0e06e9da46894c1 2024-11-23T16:14:37,986 DEBUG [StoreCloser-TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/B/132c7e3c8f734abb862221695a55b037 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/B/132c7e3c8f734abb862221695a55b037 2024-11-23T16:14:37,987 DEBUG [StoreCloser-TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/B/e2a75731746a43719653ec1be70a2a2b to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/B/e2a75731746a43719653ec1be70a2a2b 2024-11-23T16:14:37,988 DEBUG [StoreCloser-TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/B/66abafe5050742da819745e24a2ae177 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/B/66abafe5050742da819745e24a2ae177 2024-11-23T16:14:37,989 DEBUG [StoreCloser-TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/B/cb03383e6607400c8e097af59514a54a to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/B/cb03383e6607400c8e097af59514a54a 2024-11-23T16:14:37,990 DEBUG [StoreCloser-TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/B/bee71088d6ba4c8e99248cef4a1e1ca5 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/B/bee71088d6ba4c8e99248cef4a1e1ca5 2024-11-23T16:14:37,990 DEBUG [StoreCloser-TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/B/66dee4c6ddbf4593a032fca6ec7c2e6b to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/B/66dee4c6ddbf4593a032fca6ec7c2e6b 2024-11-23T16:14:37,991 DEBUG [StoreCloser-TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/B/01c7c21da7c244dfaa73f9591258d0be to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/B/01c7c21da7c244dfaa73f9591258d0be 2024-11-23T16:14:37,992 DEBUG [StoreCloser-TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/B/1174617488a94552afa9e1a3694e44a1 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/B/1174617488a94552afa9e1a3694e44a1 2024-11-23T16:14:37,993 DEBUG [StoreCloser-TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/B/9b7fef690e4449549e61a0f00cca7b9f to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/B/9b7fef690e4449549e61a0f00cca7b9f 2024-11-23T16:14:37,994 DEBUG [StoreCloser-TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/B/33a0fe6442754c2d96c63a1878c016a1 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/B/33a0fe6442754c2d96c63a1878c016a1 2024-11-23T16:14:37,995 DEBUG [StoreCloser-TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/C/5bf42799999d403d8173ae3cd6a75c07, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/C/14c9ba67b3cd4082bdd8dab406b10fbb, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/C/3a50e69226a54b7b804803c663dc3c63, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/C/709f26037d9e498db94a1930f09e167b, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/C/9c36aa9387174758a5218a7e1dfdf61e, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/C/d12fb284d6c142db95dd0da11d15d56b, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/C/cf130f6f9a254c50a6b9c4ebad74e11f, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/C/2783233a743c4d6bbb2221923dc24c3e, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/C/7ded1e012fcf4760a8d6c12379c832f3, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/C/042432554d5e4ca0b7c263ac62dcdccc, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/C/09b86ecd3ab346fb95ae235103b74167, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/C/3ee5535d7f49438f9e13c9f43178ffb3, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/C/deec6a92bb0f4215b405bcfd187e95c9, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/C/09458dc1927d49249fb9ce5f661bb94d, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/C/4279c0d888624129b950c7373b30fca9, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/C/2a3cf60e86814ce5b789f782967c3513, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/C/db8361a1cb654eb9aa52e87c1563755b, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/C/e7d9fc849d914806960662ebd8e05aef, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/C/410b96d8455449598ed19d2ed3f23b89, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/C/c17eee5581b0434abce55240de5b4ebb, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/C/fb9d1f5f1e1e4fd8a3dd40cbe8dc2796, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/C/84dc627c32b948c9aaf05a80aa818018, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/C/0663c9ac1a9140369c8e278c38a74468, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/C/a74a14f34a3542adb828cfefbebf05d9, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/C/255627e4041f4998861702bde7d081fa] to archive 2024-11-23T16:14:37,996 DEBUG [StoreCloser-TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-23T16:14:37,997 DEBUG [StoreCloser-TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/C/5bf42799999d403d8173ae3cd6a75c07 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/C/5bf42799999d403d8173ae3cd6a75c07 2024-11-23T16:14:37,998 DEBUG [StoreCloser-TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/C/14c9ba67b3cd4082bdd8dab406b10fbb to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/C/14c9ba67b3cd4082bdd8dab406b10fbb 2024-11-23T16:14:37,999 DEBUG [StoreCloser-TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/C/3a50e69226a54b7b804803c663dc3c63 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/C/3a50e69226a54b7b804803c663dc3c63 2024-11-23T16:14:38,000 DEBUG [StoreCloser-TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/C/709f26037d9e498db94a1930f09e167b to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/C/709f26037d9e498db94a1930f09e167b 2024-11-23T16:14:38,001 DEBUG [StoreCloser-TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/C/9c36aa9387174758a5218a7e1dfdf61e to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/C/9c36aa9387174758a5218a7e1dfdf61e 2024-11-23T16:14:38,002 DEBUG [StoreCloser-TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/C/d12fb284d6c142db95dd0da11d15d56b to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/C/d12fb284d6c142db95dd0da11d15d56b 2024-11-23T16:14:38,003 DEBUG [StoreCloser-TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/C/cf130f6f9a254c50a6b9c4ebad74e11f to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/C/cf130f6f9a254c50a6b9c4ebad74e11f 2024-11-23T16:14:38,004 DEBUG [StoreCloser-TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/C/2783233a743c4d6bbb2221923dc24c3e to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/C/2783233a743c4d6bbb2221923dc24c3e 2024-11-23T16:14:38,005 DEBUG [StoreCloser-TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/C/7ded1e012fcf4760a8d6c12379c832f3 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/C/7ded1e012fcf4760a8d6c12379c832f3 2024-11-23T16:14:38,006 DEBUG [StoreCloser-TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/C/042432554d5e4ca0b7c263ac62dcdccc to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/C/042432554d5e4ca0b7c263ac62dcdccc 2024-11-23T16:14:38,007 DEBUG [StoreCloser-TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/C/09b86ecd3ab346fb95ae235103b74167 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/C/09b86ecd3ab346fb95ae235103b74167 2024-11-23T16:14:38,008 DEBUG [StoreCloser-TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/C/3ee5535d7f49438f9e13c9f43178ffb3 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/C/3ee5535d7f49438f9e13c9f43178ffb3 2024-11-23T16:14:38,009 DEBUG [StoreCloser-TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/C/deec6a92bb0f4215b405bcfd187e95c9 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/C/deec6a92bb0f4215b405bcfd187e95c9 2024-11-23T16:14:38,010 DEBUG [StoreCloser-TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/C/09458dc1927d49249fb9ce5f661bb94d to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/C/09458dc1927d49249fb9ce5f661bb94d 2024-11-23T16:14:38,011 DEBUG [StoreCloser-TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/C/4279c0d888624129b950c7373b30fca9 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/C/4279c0d888624129b950c7373b30fca9 2024-11-23T16:14:38,012 DEBUG [StoreCloser-TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/C/2a3cf60e86814ce5b789f782967c3513 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/C/2a3cf60e86814ce5b789f782967c3513 2024-11-23T16:14:38,013 DEBUG [StoreCloser-TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/C/db8361a1cb654eb9aa52e87c1563755b to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/C/db8361a1cb654eb9aa52e87c1563755b 2024-11-23T16:14:38,014 DEBUG [StoreCloser-TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/C/e7d9fc849d914806960662ebd8e05aef to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/C/e7d9fc849d914806960662ebd8e05aef 2024-11-23T16:14:38,015 DEBUG [StoreCloser-TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/C/410b96d8455449598ed19d2ed3f23b89 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/C/410b96d8455449598ed19d2ed3f23b89 2024-11-23T16:14:38,016 DEBUG [StoreCloser-TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/C/c17eee5581b0434abce55240de5b4ebb to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/C/c17eee5581b0434abce55240de5b4ebb 2024-11-23T16:14:38,017 DEBUG [StoreCloser-TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/C/fb9d1f5f1e1e4fd8a3dd40cbe8dc2796 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/C/fb9d1f5f1e1e4fd8a3dd40cbe8dc2796 2024-11-23T16:14:38,018 DEBUG [StoreCloser-TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/C/84dc627c32b948c9aaf05a80aa818018 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/C/84dc627c32b948c9aaf05a80aa818018 2024-11-23T16:14:38,019 DEBUG [StoreCloser-TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/C/0663c9ac1a9140369c8e278c38a74468 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/C/0663c9ac1a9140369c8e278c38a74468 2024-11-23T16:14:38,020 DEBUG [StoreCloser-TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/C/a74a14f34a3542adb828cfefbebf05d9 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/C/a74a14f34a3542adb828cfefbebf05d9 2024-11-23T16:14:38,021 DEBUG [StoreCloser-TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/C/255627e4041f4998861702bde7d081fa to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/C/255627e4041f4998861702bde7d081fa 2024-11-23T16:14:38,025 DEBUG [RS_CLOSE_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/recovered.edits/407.seqid, newMaxSeqId=407, maxSeqId=4 2024-11-23T16:14:38,026 INFO [RS_CLOSE_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853. 2024-11-23T16:14:38,026 DEBUG [RS_CLOSE_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(1676): Region close journal for cf5d4648fde06cb520ace381e0be9853: Waiting for close lock at 1732378477005Running coprocessor pre-close hooks at 1732378477005Disabling compacts and flushes for region at 1732378477005Disabling writes for close at 1732378477005Obtaining lock to block concurrent updates at 1732378477005Preparing flush snapshotting stores in cf5d4648fde06cb520ace381e0be9853 at 1732378477005Finished memstore snapshotting TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1732378477006 (+1 ms)Flushing stores of TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853. at 1732378477007 (+1 ms)Flushing cf5d4648fde06cb520ace381e0be9853/A: creating writer at 1732378477007Flushing mob file cf5d4648fde06cb520ace381e0be9853/A: appending metadata at 1732378477014 (+7 ms)Flushing mob file cf5d4648fde06cb520ace381e0be9853/A: closing flushed file at 1732378477014Flushing cf5d4648fde06cb520ace381e0be9853/A: appending metadata at 1732378477460 (+446 ms)Flushing cf5d4648fde06cb520ace381e0be9853/A: closing flushed file at 1732378477460Flushing cf5d4648fde06cb520ace381e0be9853/B: creating writer at 1732378477478 (+18 ms)Flushing cf5d4648fde06cb520ace381e0be9853/B: appending metadata at 1732378477481 (+3 ms)Flushing cf5d4648fde06cb520ace381e0be9853/B: closing flushed file at 1732378477481Flushing cf5d4648fde06cb520ace381e0be9853/C: creating writer at 1732378477497 (+16 ms)Flushing cf5d4648fde06cb520ace381e0be9853/C: appending metadata at 1732378477507 (+10 ms)Flushing cf5d4648fde06cb520ace381e0be9853/C: closing flushed file at 1732378477507Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@956dac: reopening flushed file at 1732378477920 (+413 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@37ac3b99: reopening flushed file at 1732378477926 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@778e88f7: reopening flushed file at 1732378477932 (+6 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for cf5d4648fde06cb520ace381e0be9853 in 933ms, sequenceid=404, compaction requested=true at 1732378477938 (+6 ms)Writing region close event to WAL at 1732378478022 (+84 ms)Running coprocessor post-close hooks at 1732378478026 (+4 ms)Closed at 1732378478026 2024-11-23T16:14:38,028 INFO [RS_CLOSE_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] handler.UnassignRegionHandler(157): Closed cf5d4648fde06cb520ace381e0be9853 2024-11-23T16:14:38,029 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=56 updating hbase:meta row=cf5d4648fde06cb520ace381e0be9853, regionState=CLOSED 2024-11-23T16:14:38,030 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=57, ppid=56, state=RUNNABLE, hasLock=false; CloseRegionProcedure cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 because future has completed 2024-11-23T16:14:38,032 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=57, resume processing ppid=56 2024-11-23T16:14:38,032 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=57, ppid=56, state=SUCCESS, hasLock=false; CloseRegionProcedure cf5d4648fde06cb520ace381e0be9853, server=1a7c5470b472,44715,1732378420375 in 1.1790 sec 2024-11-23T16:14:38,033 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=56, resume processing ppid=55 2024-11-23T16:14:38,033 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=56, ppid=55, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=cf5d4648fde06cb520ace381e0be9853, UNASSIGN in 1.1830 sec 2024-11-23T16:14:38,035 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=55, resume processing ppid=54 2024-11-23T16:14:38,035 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=55, ppid=54, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.1860 sec 2024-11-23T16:14:38,036 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732378478035"}]},"ts":"1732378478035"} 2024-11-23T16:14:38,037 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-23T16:14:38,037 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-23T16:14:38,038 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=54, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.1960 sec 2024-11-23T16:14:38,630 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-23T16:14:38,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-23T16:14:38,988 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-23T16:14:38,989 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-23T16:14:38,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] procedure2.ProcedureExecutor(1139): Stored pid=58, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-23T16:14:38,991 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=58, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-23T16:14:38,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-23T16:14:38,992 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=58, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-23T16:14:38,995 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853 2024-11-23T16:14:38,998 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/A, FileablePath, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/B, FileablePath, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/C, FileablePath, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/recovered.edits] 2024-11-23T16:14:39,001 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/A/6859e4c03a244fec843582e0deaf9d37 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/A/6859e4c03a244fec843582e0deaf9d37 2024-11-23T16:14:39,005 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/A/b06c9f4c58304a0d8dd2f12d433782fb to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/A/b06c9f4c58304a0d8dd2f12d433782fb 2024-11-23T16:14:39,007 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/A/ef493ade9f9b4566b0592f39947a1276 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/A/ef493ade9f9b4566b0592f39947a1276 2024-11-23T16:14:39,013 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/B/02563ea2ab644c9090436672bee3193d to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/B/02563ea2ab644c9090436672bee3193d 2024-11-23T16:14:39,014 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/B/630b68fad74a49db8e1c70dde5222cb4 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/B/630b68fad74a49db8e1c70dde5222cb4 2024-11-23T16:14:39,016 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/B/ea3a4059fb48448aac7da181f4d01bcb to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/B/ea3a4059fb48448aac7da181f4d01bcb 2024-11-23T16:14:39,019 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/C/65711d740480460bb754bdaa626542ba to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/C/65711d740480460bb754bdaa626542ba 2024-11-23T16:14:39,022 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/C/65ed4ba4b704414f8ea6e20091576b02 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/C/65ed4ba4b704414f8ea6e20091576b02 2024-11-23T16:14:39,023 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/C/c7579036ccf84bf5ad383446f8c84c89 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/C/c7579036ccf84bf5ad383446f8c84c89 2024-11-23T16:14:39,026 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/recovered.edits/407.seqid to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853/recovered.edits/407.seqid 2024-11-23T16:14:39,027 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/cf5d4648fde06cb520ace381e0be9853 2024-11-23T16:14:39,027 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-23T16:14:39,028 DEBUG [PEWorker-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-23T16:14:39,029 DEBUG [PEWorker-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-23T16:14:39,035 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123092ad95a6cf841cabe5693002f0cf765_cf5d4648fde06cb520ace381e0be9853 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123092ad95a6cf841cabe5693002f0cf765_cf5d4648fde06cb520ace381e0be9853 2024-11-23T16:14:39,036 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112309de6b870b6c4a11b02106a02e0c0ae8_cf5d4648fde06cb520ace381e0be9853 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112309de6b870b6c4a11b02106a02e0c0ae8_cf5d4648fde06cb520ace381e0be9853 2024-11-23T16:14:39,038 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411231de759dc86364cf691ee759b4d9ccec4_cf5d4648fde06cb520ace381e0be9853 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411231de759dc86364cf691ee759b4d9ccec4_cf5d4648fde06cb520ace381e0be9853 2024-11-23T16:14:39,039 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411233f731c364cfb4ac980412f1b2bfbdf59_cf5d4648fde06cb520ace381e0be9853 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411233f731c364cfb4ac980412f1b2bfbdf59_cf5d4648fde06cb520ace381e0be9853 2024-11-23T16:14:39,040 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112346db5400d0c94326ac5184a2f680320e_cf5d4648fde06cb520ace381e0be9853 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112346db5400d0c94326ac5184a2f680320e_cf5d4648fde06cb520ace381e0be9853 2024-11-23T16:14:39,042 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123475a609c00be404e812c6939ed83f7fb_cf5d4648fde06cb520ace381e0be9853 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123475a609c00be404e812c6939ed83f7fb_cf5d4648fde06cb520ace381e0be9853 2024-11-23T16:14:39,043 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411234c8ac4c07c6d4aeeb6d98599c6c922d5_cf5d4648fde06cb520ace381e0be9853 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411234c8ac4c07c6d4aeeb6d98599c6c922d5_cf5d4648fde06cb520ace381e0be9853 2024-11-23T16:14:39,045 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411235b2ab43559314eafb9ebf2d7b1b45286_cf5d4648fde06cb520ace381e0be9853 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411235b2ab43559314eafb9ebf2d7b1b45286_cf5d4648fde06cb520ace381e0be9853 2024-11-23T16:14:39,046 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411235f6b7ad87d124dcdb65f108724ce0548_cf5d4648fde06cb520ace381e0be9853 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411235f6b7ad87d124dcdb65f108724ce0548_cf5d4648fde06cb520ace381e0be9853 2024-11-23T16:14:39,047 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123856b072d3dd144d38c42abca4fdfc043_cf5d4648fde06cb520ace381e0be9853 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123856b072d3dd144d38c42abca4fdfc043_cf5d4648fde06cb520ace381e0be9853 2024-11-23T16:14:39,050 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112396b76e6ebded44d79dbddef7e93f2e94_cf5d4648fde06cb520ace381e0be9853 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112396b76e6ebded44d79dbddef7e93f2e94_cf5d4648fde06cb520ace381e0be9853 2024-11-23T16:14:39,051 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411239e1dd22508334809b5d7df8dc91e5699_cf5d4648fde06cb520ace381e0be9853 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411239e1dd22508334809b5d7df8dc91e5699_cf5d4648fde06cb520ace381e0be9853 2024-11-23T16:14:39,053 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123a10252a8bf824235a8a7119cdcade2c3_cf5d4648fde06cb520ace381e0be9853 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123a10252a8bf824235a8a7119cdcade2c3_cf5d4648fde06cb520ace381e0be9853 2024-11-23T16:14:39,054 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123b592ad9a15284f189537a0f832052560_cf5d4648fde06cb520ace381e0be9853 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123b592ad9a15284f189537a0f832052560_cf5d4648fde06cb520ace381e0be9853 2024-11-23T16:14:39,056 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123c360bbed6efa42e783bbe9da99e487ca_cf5d4648fde06cb520ace381e0be9853 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123c360bbed6efa42e783bbe9da99e487ca_cf5d4648fde06cb520ace381e0be9853 2024-11-23T16:14:39,058 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123d6711afba503466995afc6582c60620f_cf5d4648fde06cb520ace381e0be9853 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123d6711afba503466995afc6582c60620f_cf5d4648fde06cb520ace381e0be9853 2024-11-23T16:14:39,060 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123e48a7e6d27ae41cf9bae65a2fb3db1a5_cf5d4648fde06cb520ace381e0be9853 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123e48a7e6d27ae41cf9bae65a2fb3db1a5_cf5d4648fde06cb520ace381e0be9853 2024-11-23T16:14:39,061 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123e548afcfc9214176ad55568064585f18_cf5d4648fde06cb520ace381e0be9853 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123e548afcfc9214176ad55568064585f18_cf5d4648fde06cb520ace381e0be9853 2024-11-23T16:14:39,062 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123e8d265743a374694b89f4e1209acaa82_cf5d4648fde06cb520ace381e0be9853 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123e8d265743a374694b89f4e1209acaa82_cf5d4648fde06cb520ace381e0be9853 2024-11-23T16:14:39,063 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123f0ab74254bad4613981a072935329952_cf5d4648fde06cb520ace381e0be9853 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123f0ab74254bad4613981a072935329952_cf5d4648fde06cb520ace381e0be9853 2024-11-23T16:14:39,064 DEBUG [PEWorker-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-23T16:14:39,066 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=58, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-23T16:14:39,085 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-23T16:14:39,088 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-23T16:14:39,089 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=58, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-23T16:14:39,090 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-23T16:14:39,093 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732378479090"}]},"ts":"9223372036854775807"} 2024-11-23T16:14:39,097 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-23T16:14:39,097 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => cf5d4648fde06cb520ace381e0be9853, NAME => 'TestAcidGuarantees,,1732378449932.cf5d4648fde06cb520ace381e0be9853.', STARTKEY => '', ENDKEY => ''}] 2024-11-23T16:14:39,097 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-23T16:14:39,097 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732378479097"}]},"ts":"9223372036854775807"} 2024-11-23T16:14:39,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-23T16:14:39,100 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-23T16:14:39,100 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=58, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-23T16:14:39,102 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 111 msec 2024-11-23T16:14:39,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-23T16:14:39,309 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-23T16:14:39,309 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-23T16:14:39,326 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobMixedAtomicity Thread=249 (was 248) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-2057066486_22 at /127.0.0.1:39972 [Waiting for operation #15] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2703c80e-a8f3-3f6b-fe54-0323a389f54b/cluster_d53936a7-f36f-e8d1-b808-86c188667d37/data/data1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2703c80e-a8f3-3f6b-fe54-0323a389f54b/cluster_d53936a7-f36f-e8d1-b808-86c188667d37/data/data2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=457 (was 453) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=435 (was 354) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=8070 (was 7987) - AvailableMemoryMB LEAK? - 2024-11-23T16:14:39,337 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testGetAtomicity Thread=249, OpenFileDescriptor=457, MaxFileDescriptor=1048576, SystemLoadAverage=435, ProcessCount=11, AvailableMemoryMB=8069 2024-11-23T16:14:39,338 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-23T16:14:39,339 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-23T16:14:39,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] procedure2.ProcedureExecutor(1139): Stored pid=59, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-23T16:14:39,341 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=59, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-23T16:14:39,341 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:39,341 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 59 2024-11-23T16:14:39,342 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=59, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-23T16:14:39,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=59 2024-11-23T16:14:39,349 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742060_1236 (size=1061) 2024-11-23T16:14:39,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=59 2024-11-23T16:14:39,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=59 2024-11-23T16:14:39,752 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 5b34e731541fe76c476f23c5dc5aa28e, NAME => 'TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca 2024-11-23T16:14:39,760 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742061_1237 (size=53) 2024-11-23T16:14:39,766 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T16:14:39,766 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 5b34e731541fe76c476f23c5dc5aa28e, disabling compactions & flushes 2024-11-23T16:14:39,766 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e. 2024-11-23T16:14:39,766 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e. 2024-11-23T16:14:39,766 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e. after waiting 0 ms 2024-11-23T16:14:39,766 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e. 2024-11-23T16:14:39,766 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e. 2024-11-23T16:14:39,766 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 5b34e731541fe76c476f23c5dc5aa28e: Waiting for close lock at 1732378479766Disabling compacts and flushes for region at 1732378479766Disabling writes for close at 1732378479766Writing region close event to WAL at 1732378479766Closed at 1732378479766 2024-11-23T16:14:39,767 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=59, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-23T16:14:39,768 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732378479767"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732378479767"}]},"ts":"1732378479767"} 2024-11-23T16:14:39,770 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-23T16:14:39,771 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=59, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-23T16:14:39,771 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732378479771"}]},"ts":"1732378479771"} 2024-11-23T16:14:39,774 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-23T16:14:39,774 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=60, ppid=59, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5b34e731541fe76c476f23c5dc5aa28e, ASSIGN}] 2024-11-23T16:14:39,775 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=60, ppid=59, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5b34e731541fe76c476f23c5dc5aa28e, ASSIGN 2024-11-23T16:14:39,777 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=60, ppid=59, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=5b34e731541fe76c476f23c5dc5aa28e, ASSIGN; state=OFFLINE, location=1a7c5470b472,44715,1732378420375; forceNewPlan=false, retain=false 2024-11-23T16:14:39,928 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=60 updating hbase:meta row=5b34e731541fe76c476f23c5dc5aa28e, regionState=OPENING, regionLocation=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:39,930 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=60, ppid=59, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=5b34e731541fe76c476f23c5dc5aa28e, ASSIGN because future has completed 2024-11-23T16:14:39,930 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=61, ppid=60, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375}] 2024-11-23T16:14:39,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=59 2024-11-23T16:14:40,088 INFO [RS_OPEN_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e. 2024-11-23T16:14:40,088 DEBUG [RS_OPEN_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(7752): Opening region: {ENCODED => 5b34e731541fe76c476f23c5dc5aa28e, NAME => 'TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e.', STARTKEY => '', ENDKEY => ''} 2024-11-23T16:14:40,088 DEBUG [RS_OPEN_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 5b34e731541fe76c476f23c5dc5aa28e 2024-11-23T16:14:40,088 DEBUG [RS_OPEN_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T16:14:40,088 DEBUG [RS_OPEN_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(7794): checking encryption for 5b34e731541fe76c476f23c5dc5aa28e 2024-11-23T16:14:40,088 DEBUG [RS_OPEN_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(7797): checking classloading for 5b34e731541fe76c476f23c5dc5aa28e 2024-11-23T16:14:40,095 INFO [StoreOpener-5b34e731541fe76c476f23c5dc5aa28e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 5b34e731541fe76c476f23c5dc5aa28e 2024-11-23T16:14:40,096 INFO [StoreOpener-5b34e731541fe76c476f23c5dc5aa28e-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-23T16:14:40,097 INFO [StoreOpener-5b34e731541fe76c476f23c5dc5aa28e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5b34e731541fe76c476f23c5dc5aa28e columnFamilyName A 2024-11-23T16:14:40,097 DEBUG [StoreOpener-5b34e731541fe76c476f23c5dc5aa28e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:40,097 INFO [StoreOpener-5b34e731541fe76c476f23c5dc5aa28e-1 {}] regionserver.HStore(327): Store=5b34e731541fe76c476f23c5dc5aa28e/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T16:14:40,097 INFO [StoreOpener-5b34e731541fe76c476f23c5dc5aa28e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 5b34e731541fe76c476f23c5dc5aa28e 2024-11-23T16:14:40,098 INFO [StoreOpener-5b34e731541fe76c476f23c5dc5aa28e-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-23T16:14:40,098 INFO [StoreOpener-5b34e731541fe76c476f23c5dc5aa28e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5b34e731541fe76c476f23c5dc5aa28e columnFamilyName B 2024-11-23T16:14:40,098 DEBUG [StoreOpener-5b34e731541fe76c476f23c5dc5aa28e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:40,099 INFO [StoreOpener-5b34e731541fe76c476f23c5dc5aa28e-1 {}] regionserver.HStore(327): Store=5b34e731541fe76c476f23c5dc5aa28e/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T16:14:40,099 INFO [StoreOpener-5b34e731541fe76c476f23c5dc5aa28e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 5b34e731541fe76c476f23c5dc5aa28e 2024-11-23T16:14:40,100 INFO [StoreOpener-5b34e731541fe76c476f23c5dc5aa28e-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-23T16:14:40,100 INFO [StoreOpener-5b34e731541fe76c476f23c5dc5aa28e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5b34e731541fe76c476f23c5dc5aa28e columnFamilyName C 2024-11-23T16:14:40,100 DEBUG [StoreOpener-5b34e731541fe76c476f23c5dc5aa28e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:14:40,100 INFO [StoreOpener-5b34e731541fe76c476f23c5dc5aa28e-1 {}] regionserver.HStore(327): Store=5b34e731541fe76c476f23c5dc5aa28e/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T16:14:40,101 INFO [RS_OPEN_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e. 2024-11-23T16:14:40,101 DEBUG [RS_OPEN_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1038): replaying wal for 5b34e731541fe76c476f23c5dc5aa28e 2024-11-23T16:14:40,101 DEBUG [RS_OPEN_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e 2024-11-23T16:14:40,102 DEBUG [RS_OPEN_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e 2024-11-23T16:14:40,103 DEBUG [RS_OPEN_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1048): stopping wal replay for 5b34e731541fe76c476f23c5dc5aa28e 2024-11-23T16:14:40,103 DEBUG [RS_OPEN_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1060): Cleaning up temporary data for 5b34e731541fe76c476f23c5dc5aa28e 2024-11-23T16:14:40,103 DEBUG [RS_OPEN_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-23T16:14:40,104 DEBUG [RS_OPEN_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1093): writing seq id for 5b34e731541fe76c476f23c5dc5aa28e 2024-11-23T16:14:40,107 DEBUG [RS_OPEN_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-23T16:14:40,107 INFO [RS_OPEN_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1114): Opened 5b34e731541fe76c476f23c5dc5aa28e; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72631553, jitterRate=0.08229447901248932}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-23T16:14:40,107 DEBUG [RS_OPEN_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 5b34e731541fe76c476f23c5dc5aa28e 2024-11-23T16:14:40,108 DEBUG [RS_OPEN_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1006): Region open journal for 5b34e731541fe76c476f23c5dc5aa28e: Running coprocessor pre-open hook at 1732378480088Writing region info on filesystem at 1732378480089 (+1 ms)Initializing all the Stores at 1732378480089Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732378480089Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732378480095 (+6 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732378480095Cleaning up temporary data from old regions at 1732378480103 (+8 ms)Running coprocessor post-open hooks at 1732378480107 (+4 ms)Region opened successfully at 1732378480108 (+1 ms) 2024-11-23T16:14:40,109 INFO [RS_OPEN_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., pid=61, masterSystemTime=1732378480082 2024-11-23T16:14:40,111 DEBUG [RS_OPEN_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e. 2024-11-23T16:14:40,111 INFO [RS_OPEN_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e. 2024-11-23T16:14:40,112 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=60 updating hbase:meta row=5b34e731541fe76c476f23c5dc5aa28e, regionState=OPEN, openSeqNum=2, regionLocation=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:40,114 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=61, ppid=60, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 because future has completed 2024-11-23T16:14:40,117 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=61, resume processing ppid=60 2024-11-23T16:14:40,117 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 in 185 msec 2024-11-23T16:14:40,119 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=60, resume processing ppid=59 2024-11-23T16:14:40,119 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=60, ppid=59, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5b34e731541fe76c476f23c5dc5aa28e, ASSIGN in 343 msec 2024-11-23T16:14:40,120 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=59, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-23T16:14:40,120 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732378480120"}]},"ts":"1732378480120"} 2024-11-23T16:14:40,121 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-23T16:14:40,122 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=59, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-23T16:14:40,124 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 783 msec 2024-11-23T16:14:40,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=59 2024-11-23T16:14:40,479 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-23T16:14:40,480 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3fe11fca, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T16:14:40,480 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1a7c5470b472,45799,-1 for getting cluster id 2024-11-23T16:14:40,480 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-23T16:14:40,481 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '34e0592b-548a-4d73-be6a-df1bafadfd92' 2024-11-23T16:14:40,481 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-23T16:14:40,481 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "34e0592b-548a-4d73-be6a-df1bafadfd92" 2024-11-23T16:14:40,481 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@78c149b4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T16:14:40,481 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1a7c5470b472,45799,-1] 2024-11-23T16:14:40,482 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-23T16:14:40,482 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T16:14:40,483 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58072, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-23T16:14:40,483 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4dbbbcaa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T16:14:40,483 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T16:14:40,484 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1a7c5470b472,44715,1732378420375, seqNum=-1] 2024-11-23T16:14:40,484 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T16:14:40,485 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50412, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T16:14:40,486 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-23T16:14:40,486 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncConnectionImpl(321): The fetched master address is 1a7c5470b472,45799,1732378419617 2024-11-23T16:14:40,486 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@44acbc3e 2024-11-23T16:14:40,487 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-23T16:14:40,487 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58076, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-23T16:14:40,489 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@34b5b273, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T16:14:40,489 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1a7c5470b472,45799,-1 for getting cluster id 2024-11-23T16:14:40,489 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-23T16:14:40,489 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '34e0592b-548a-4d73-be6a-df1bafadfd92' 2024-11-23T16:14:40,489 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-23T16:14:40,490 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "34e0592b-548a-4d73-be6a-df1bafadfd92" 2024-11-23T16:14:40,490 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@44220a59, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T16:14:40,490 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1a7c5470b472,45799,-1] 2024-11-23T16:14:40,490 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-23T16:14:40,490 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T16:14:40,490 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58090, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-23T16:14:40,491 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@46b9aca9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T16:14:40,492 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1240be3c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T16:14:40,492 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1a7c5470b472,45799,-1 for getting cluster id 2024-11-23T16:14:40,492 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-23T16:14:40,492 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '34e0592b-548a-4d73-be6a-df1bafadfd92' 2024-11-23T16:14:40,492 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-23T16:14:40,493 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "34e0592b-548a-4d73-be6a-df1bafadfd92" 2024-11-23T16:14:40,493 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2d59b38d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T16:14:40,493 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1a7c5470b472,45799,-1] 2024-11-23T16:14:40,493 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-23T16:14:40,493 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T16:14:40,494 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58106, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-23T16:14:40,494 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5cbd2793, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T16:14:40,495 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2885eb44, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T16:14:40,495 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1a7c5470b472,45799,-1 for getting cluster id 2024-11-23T16:14:40,495 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-23T16:14:40,495 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '34e0592b-548a-4d73-be6a-df1bafadfd92' 2024-11-23T16:14:40,495 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-23T16:14:40,496 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "34e0592b-548a-4d73-be6a-df1bafadfd92" 2024-11-23T16:14:40,496 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3f176062, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T16:14:40,496 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1a7c5470b472,45799,-1] 2024-11-23T16:14:40,496 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-23T16:14:40,496 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T16:14:40,497 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58118, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-23T16:14:40,497 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@23c0712f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T16:14:40,498 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@692a6227, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T16:14:40,498 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1a7c5470b472,45799,-1 for getting cluster id 2024-11-23T16:14:40,498 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-23T16:14:40,498 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '34e0592b-548a-4d73-be6a-df1bafadfd92' 2024-11-23T16:14:40,498 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-23T16:14:40,498 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "34e0592b-548a-4d73-be6a-df1bafadfd92" 2024-11-23T16:14:40,499 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@76d855da, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T16:14:40,499 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1a7c5470b472,45799,-1] 2024-11-23T16:14:40,499 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-23T16:14:40,499 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T16:14:40,499 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58148, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-23T16:14:40,500 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2ad5b2a2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T16:14:40,500 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@128d97bc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T16:14:40,500 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1a7c5470b472,45799,-1 for getting cluster id 2024-11-23T16:14:40,500 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-23T16:14:40,501 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '34e0592b-548a-4d73-be6a-df1bafadfd92' 2024-11-23T16:14:40,501 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-23T16:14:40,501 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "34e0592b-548a-4d73-be6a-df1bafadfd92" 2024-11-23T16:14:40,501 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@631ca1bc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T16:14:40,501 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1a7c5470b472,45799,-1] 2024-11-23T16:14:40,502 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-23T16:14:40,502 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T16:14:40,502 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58166, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-23T16:14:40,503 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4008b168, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T16:14:40,503 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@626de388, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T16:14:40,503 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1a7c5470b472,45799,-1 for getting cluster id 2024-11-23T16:14:40,503 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-23T16:14:40,504 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '34e0592b-548a-4d73-be6a-df1bafadfd92' 2024-11-23T16:14:40,504 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-23T16:14:40,504 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "34e0592b-548a-4d73-be6a-df1bafadfd92" 2024-11-23T16:14:40,504 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2c053632, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T16:14:40,504 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1a7c5470b472,45799,-1] 2024-11-23T16:14:40,505 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-23T16:14:40,505 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T16:14:40,505 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58184, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-23T16:14:40,506 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a636371, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T16:14:40,506 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@47a5c877, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T16:14:40,506 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1a7c5470b472,45799,-1 for getting cluster id 2024-11-23T16:14:40,507 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-23T16:14:40,507 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '34e0592b-548a-4d73-be6a-df1bafadfd92' 2024-11-23T16:14:40,507 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-23T16:14:40,507 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "34e0592b-548a-4d73-be6a-df1bafadfd92" 2024-11-23T16:14:40,508 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@79a6e548, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T16:14:40,508 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1a7c5470b472,45799,-1] 2024-11-23T16:14:40,508 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-23T16:14:40,508 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T16:14:40,509 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58204, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-23T16:14:40,509 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2c2fea3d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T16:14:40,510 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@679f7426, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T16:14:40,510 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1a7c5470b472,45799,-1 for getting cluster id 2024-11-23T16:14:40,510 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-23T16:14:40,511 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '34e0592b-548a-4d73-be6a-df1bafadfd92' 2024-11-23T16:14:40,511 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-23T16:14:40,511 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "34e0592b-548a-4d73-be6a-df1bafadfd92" 2024-11-23T16:14:40,511 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@42283f50, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T16:14:40,511 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1a7c5470b472,45799,-1] 2024-11-23T16:14:40,511 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-23T16:14:40,511 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T16:14:40,512 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58214, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-23T16:14:40,513 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@205bcdcc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T16:14:40,513 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b71af22, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T16:14:40,513 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1a7c5470b472,45799,-1 for getting cluster id 2024-11-23T16:14:40,513 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-23T16:14:40,514 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '34e0592b-548a-4d73-be6a-df1bafadfd92' 2024-11-23T16:14:40,514 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-23T16:14:40,514 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "34e0592b-548a-4d73-be6a-df1bafadfd92" 2024-11-23T16:14:40,514 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@61dc1345, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T16:14:40,515 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1a7c5470b472,45799,-1] 2024-11-23T16:14:40,515 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-23T16:14:40,515 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T16:14:40,516 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58236, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-23T16:14:40,516 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1a6d3495, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T16:14:40,517 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@728d338, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T16:14:40,517 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1a7c5470b472,45799,-1 for getting cluster id 2024-11-23T16:14:40,517 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-23T16:14:40,518 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '34e0592b-548a-4d73-be6a-df1bafadfd92' 2024-11-23T16:14:40,518 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-23T16:14:40,518 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "34e0592b-548a-4d73-be6a-df1bafadfd92" 2024-11-23T16:14:40,518 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@71d997e9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T16:14:40,518 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1a7c5470b472,45799,-1] 2024-11-23T16:14:40,518 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-23T16:14:40,518 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T16:14:40,519 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58240, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-23T16:14:40,520 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6846fe5f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T16:14:40,520 DEBUG [Thread-1111 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T16:14:40,521 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-23T16:14:40,521 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1a7c5470b472,44715,1732378420375, seqNum=-1] 2024-11-23T16:14:40,522 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T16:14:40,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] procedure2.ProcedureExecutor(1139): Stored pid=62, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees 2024-11-23T16:14:40,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-23T16:14:40,523 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=62, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-23T16:14:40,523 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50424, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T16:14:40,524 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=62, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-23T16:14:40,524 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=63, ppid=62, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-23T16:14:40,525 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2] 2024-11-23T16:14:40,533 DEBUG [Thread-1118 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T16:14:40,533 DEBUG [Thread-1120 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T16:14:40,533 DEBUG [Thread-1124 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T16:14:40,534 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1a7c5470b472,44715,1732378420375, seqNum=-1] 2024-11-23T16:14:40,534 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1a7c5470b472,44715,1732378420375, seqNum=-1] 2024-11-23T16:14:40,534 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1a7c5470b472,44715,1732378420375, seqNum=-1] 2024-11-23T16:14:40,534 DEBUG [Thread-1107 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T16:14:40,534 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T16:14:40,534 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T16:14:40,534 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T16:14:40,535 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50428, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T16:14:40,536 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50438, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T16:14:40,536 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50444, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T16:14:40,536 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1a7c5470b472,44715,1732378420375, seqNum=-1] 2024-11-23T16:14:40,536 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T16:14:40,538 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2] 2024-11-23T16:14:40,538 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2] 2024-11-23T16:14:40,538 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50454, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T16:14:40,538 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2] 2024-11-23T16:14:40,539 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2] 2024-11-23T16:14:40,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(8855): Flush requested on 5b34e731541fe76c476f23c5dc5aa28e 2024-11-23T16:14:40,542 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5b34e731541fe76c476f23c5dc5aa28e 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-23T16:14:40,542 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5b34e731541fe76c476f23c5dc5aa28e, store=A 2024-11-23T16:14:40,542 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-23T16:14:40,542 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5b34e731541fe76c476f23c5dc5aa28e, store=B 2024-11-23T16:14:40,542 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-23T16:14:40,542 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5b34e731541fe76c476f23c5dc5aa28e, store=C 2024-11-23T16:14:40,542 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-23T16:14:40,555 DEBUG [Thread-1109 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T16:14:40,556 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1a7c5470b472,44715,1732378420375, seqNum=-1] 2024-11-23T16:14:40,556 DEBUG [Thread-1126 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T16:14:40,557 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T16:14:40,557 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1a7c5470b472,44715,1732378420375, seqNum=-1] 2024-11-23T16:14:40,558 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T16:14:40,558 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50462, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T16:14:40,559 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50470, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T16:14:40,559 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2] 2024-11-23T16:14:40,561 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2] 2024-11-23T16:14:40,575 DEBUG [Thread-1115 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T16:14:40,576 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1a7c5470b472,44715,1732378420375, seqNum=-1] 2024-11-23T16:14:40,576 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T16:14:40,576 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/.tmp/A/cc582adb0e4a4cc6b18911da9d4817aa is 50, key is test_row_0/A:col10/1732378480540/Put/seqid=0 2024-11-23T16:14:40,577 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50480, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T16:14:40,578 DEBUG [Thread-1113 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T16:14:40,579 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1a7c5470b472,44715,1732378420375, seqNum=-1] 2024-11-23T16:14:40,579 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T16:14:40,579 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2] 2024-11-23T16:14:40,580 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50486, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T16:14:40,582 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2] 2024-11-23T16:14:40,590 DEBUG [Thread-1122 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T16:14:40,591 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1a7c5470b472,44715,1732378420375, seqNum=-1] 2024-11-23T16:14:40,591 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T16:14:40,593 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50500, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T16:14:40,594 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742062_1238 (size=12001) 2024-11-23T16:14:40,594 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2] 2024-11-23T16:14:40,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-23T16:14:40,638 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:40,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50424 deadline: 1732378540586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:40,639 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:40,639 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:40,639 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:14:40,639 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:40,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 2 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50480 deadline: 1732378540587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:40,641 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1a7c5470b472:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=160, cellSize=7320, totalHeapSize=16880, min timestamp=1732378480572, max timestamp=1732378480590Num uniques 859; 2024-11-23T16:14:40,641 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1a7c5470b472:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=7320, suffix heap size=117064,new segment heap size=16880 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=160 2024-11-23T16:14:40,643 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1a7c5470b472:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11854, totalHeapSize=27176, min timestamp=1732378480572, max timestamp=1732378480590Num uniques 859; 2024-11-23T16:14:40,643 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1a7c5470b472:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=11854, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-23T16:14:40,643 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1a7c5470b472:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11854, totalHeapSize=27176, min timestamp=1732378480572, max timestamp=1732378480590Num uniques 859; 2024-11-23T16:14:40,643 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:40,643 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:40,643 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:14:40,643 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1a7c5470b472:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=11854, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-23T16:14:40,676 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-23T16:14:40,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e. 2024-11-23T16:14:40,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e. as already flushing 2024-11-23T16:14:40,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e. 2024-11-23T16:14:40,677 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 5b34e731541fe76c476f23c5dc5aa28e, NAME => 'TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:14:40,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 5b34e731541fe76c476f23c5dc5aa28e, NAME => 'TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:14:40,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5b34e731541fe76c476f23c5dc5aa28e, NAME => 'TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5b34e731541fe76c476f23c5dc5aa28e, NAME => 'TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:14:40,682 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:40,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50454 deadline: 1732378540675, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:40,683 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:40,683 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:40,683 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:14:40,684 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:40,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50486 deadline: 1732378540675, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:40,685 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:40,685 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:40,685 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:14:40,690 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:40,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50462 deadline: 1732378540683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:40,691 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:40,691 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:40,691 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:14:40,750 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:40,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50424 deadline: 1732378540748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:40,750 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:40,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 3 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50480 deadline: 1732378540748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:40,750 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:40,750 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:40,750 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:14:40,751 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:40,751 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:40,751 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:14:40,790 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:40,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50454 deadline: 1732378540788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:40,791 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:40,791 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:40,791 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:40,791 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:14:40,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50486 deadline: 1732378540788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:40,791 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:40,791 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:40,791 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:14:40,800 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:40,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50462 deadline: 1732378540798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:40,800 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:40,800 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:40,800 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:14:40,834 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-23T16:14:40,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e. 2024-11-23T16:14:40,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e. as already flushing 2024-11-23T16:14:40,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e. 2024-11-23T16:14:40,834 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 5b34e731541fe76c476f23c5dc5aa28e, NAME => 'TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:14:40,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 5b34e731541fe76c476f23c5dc5aa28e, NAME => 'TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:14:40,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5b34e731541fe76c476f23c5dc5aa28e, NAME => 'TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5b34e731541fe76c476f23c5dc5aa28e, NAME => 'TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:14:40,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-23T16:14:40,960 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:40,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50424 deadline: 1732378540958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:40,961 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:40,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 4 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50480 deadline: 1732378540958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:40,961 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:40,961 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:40,961 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:40,961 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:40,961 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:14:40,961 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:14:40,988 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-23T16:14:40,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e. 2024-11-23T16:14:40,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e. as already flushing 2024-11-23T16:14:40,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e. 2024-11-23T16:14:40,989 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 5b34e731541fe76c476f23c5dc5aa28e, NAME => 'TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:14:40,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 5b34e731541fe76c476f23c5dc5aa28e, NAME => 'TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:14:40,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5b34e731541fe76c476f23c5dc5aa28e, NAME => 'TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5b34e731541fe76c476f23c5dc5aa28e, NAME => 'TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:14:40,995 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/.tmp/A/cc582adb0e4a4cc6b18911da9d4817aa 2024-11-23T16:14:41,002 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:41,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50454 deadline: 1732378540998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:41,002 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:41,003 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:41,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50486 deadline: 1732378540998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:41,003 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:41,003 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:14:41,003 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:41,003 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:41,003 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:14:41,010 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:41,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50462 deadline: 1732378541009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:41,010 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:41,010 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:41,010 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:14:41,023 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/.tmp/B/09a9483ab01c4a93b5dd72f29e8976fe is 50, key is test_row_0/B:col10/1732378480540/Put/seqid=0 2024-11-23T16:14:41,043 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742063_1239 (size=12001) 2024-11-23T16:14:41,047 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/.tmp/B/09a9483ab01c4a93b5dd72f29e8976fe 2024-11-23T16:14:41,097 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/.tmp/C/011c9d7b98614d99b81fab8639fdf62d is 50, key is test_row_0/C:col10/1732378480540/Put/seqid=0 2024-11-23T16:14:41,108 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742064_1240 (size=12001) 2024-11-23T16:14:41,143 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-23T16:14:41,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e. 2024-11-23T16:14:41,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e. as already flushing 2024-11-23T16:14:41,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e. 2024-11-23T16:14:41,144 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 5b34e731541fe76c476f23c5dc5aa28e, NAME => 'TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:14:41,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 5b34e731541fe76c476f23c5dc5aa28e, NAME => 'TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:14:41,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5b34e731541fe76c476f23c5dc5aa28e, NAME => 'TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5b34e731541fe76c476f23c5dc5aa28e, NAME => 'TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:14:41,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-23T16:14:41,272 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:41,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50424 deadline: 1732378541268, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:41,273 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:41,273 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:41,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50480 deadline: 1732378541268, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:41,273 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:41,273 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:14:41,273 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:41,273 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:41,273 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:14:41,299 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-23T16:14:41,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e. 2024-11-23T16:14:41,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e. as already flushing 2024-11-23T16:14:41,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e. 2024-11-23T16:14:41,299 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 5b34e731541fe76c476f23c5dc5aa28e, NAME => 'TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:14:41,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 5b34e731541fe76c476f23c5dc5aa28e, NAME => 'TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:14:41,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5b34e731541fe76c476f23c5dc5aa28e, NAME => 'TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5b34e731541fe76c476f23c5dc5aa28e, NAME => 'TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:14:41,311 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:41,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50454 deadline: 1732378541308, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:41,311 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:41,311 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:41,311 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:41,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50486 deadline: 1732378541309, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:41,311 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:14:41,316 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:41,316 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:41,316 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:14:41,321 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:41,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50462 deadline: 1732378541318, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:41,322 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:41,322 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:41,322 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:14:41,453 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-23T16:14:41,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e. 2024-11-23T16:14:41,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e. as already flushing 2024-11-23T16:14:41,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e. 2024-11-23T16:14:41,453 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 5b34e731541fe76c476f23c5dc5aa28e, NAME => 'TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:14:41,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 5b34e731541fe76c476f23c5dc5aa28e, NAME => 'TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:14:41,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5b34e731541fe76c476f23c5dc5aa28e, NAME => 'TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5b34e731541fe76c476f23c5dc5aa28e, NAME => 'TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:14:41,510 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/.tmp/C/011c9d7b98614d99b81fab8639fdf62d 2024-11-23T16:14:41,515 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/.tmp/A/cc582adb0e4a4cc6b18911da9d4817aa as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/A/cc582adb0e4a4cc6b18911da9d4817aa 2024-11-23T16:14:41,521 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/A/cc582adb0e4a4cc6b18911da9d4817aa, entries=150, sequenceid=12, filesize=11.7 K 2024-11-23T16:14:41,522 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/.tmp/B/09a9483ab01c4a93b5dd72f29e8976fe as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/B/09a9483ab01c4a93b5dd72f29e8976fe 2024-11-23T16:14:41,526 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/B/09a9483ab01c4a93b5dd72f29e8976fe, entries=150, sequenceid=12, filesize=11.7 K 2024-11-23T16:14:41,528 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/.tmp/C/011c9d7b98614d99b81fab8639fdf62d as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/C/011c9d7b98614d99b81fab8639fdf62d 2024-11-23T16:14:41,532 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/C/011c9d7b98614d99b81fab8639fdf62d, entries=150, sequenceid=12, filesize=11.7 K 2024-11-23T16:14:41,533 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=129.75 KB/132866 for 5b34e731541fe76c476f23c5dc5aa28e in 992ms, sequenceid=12, compaction requested=false 2024-11-23T16:14:41,533 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5b34e731541fe76c476f23c5dc5aa28e: 2024-11-23T16:14:41,606 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-23T16:14:41,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e. 2024-11-23T16:14:41,607 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2902): Flushing 5b34e731541fe76c476f23c5dc5aa28e 3/3 column families, dataSize=129.75 KB heapSize=365.55 KB 2024-11-23T16:14:41,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5b34e731541fe76c476f23c5dc5aa28e, store=A 2024-11-23T16:14:41,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-23T16:14:41,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5b34e731541fe76c476f23c5dc5aa28e, store=B 2024-11-23T16:14:41,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-23T16:14:41,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5b34e731541fe76c476f23c5dc5aa28e, store=C 2024-11-23T16:14:41,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-23T16:14:41,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/.tmp/A/cf65c7a4c4994b898e5df62963e55b3e is 50, key is test_row_0/A:col10/1732378480665/Put/seqid=0 2024-11-23T16:14:41,633 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742065_1241 (size=12001) 2024-11-23T16:14:41,634 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/.tmp/A/cf65c7a4c4994b898e5df62963e55b3e 2024-11-23T16:14:41,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/.tmp/B/fbb105af53f64607977e5eb3818f7143 is 50, key is test_row_0/B:col10/1732378480665/Put/seqid=0 2024-11-23T16:14:41,654 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742066_1242 (size=12001) 2024-11-23T16:14:41,654 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/.tmp/B/fbb105af53f64607977e5eb3818f7143 2024-11-23T16:14:41,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-23T16:14:41,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/.tmp/C/22c41e52183c4f37949452f6e1234d84 is 50, key is test_row_0/C:col10/1732378480665/Put/seqid=0 2024-11-23T16:14:41,670 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742067_1243 (size=12001) 2024-11-23T16:14:41,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(8855): Flush requested on 5b34e731541fe76c476f23c5dc5aa28e 2024-11-23T16:14:41,782 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e. as already flushing 2024-11-23T16:14:41,803 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:41,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50424 deadline: 1732378541799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:41,804 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:41,804 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:41,804 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:14:41,807 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:41,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50480 deadline: 1732378541803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:41,808 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:41,808 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:41,808 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:14:41,821 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:41,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50454 deadline: 1732378541817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:41,822 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:41,822 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:41,822 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:14:41,831 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:41,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50486 deadline: 1732378541828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:41,832 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:41,832 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:41,832 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:41,832 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:14:41,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50462 deadline: 1732378541828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:41,832 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:41,832 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:41,832 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:14:41,910 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:41,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50424 deadline: 1732378541908, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:41,912 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:41,912 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:41,912 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:14:41,920 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:41,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50480 deadline: 1732378541918, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:41,921 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:41,921 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:41,921 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:14:42,071 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.30 KB at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/.tmp/C/22c41e52183c4f37949452f6e1234d84 2024-11-23T16:14:42,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/.tmp/A/cf65c7a4c4994b898e5df62963e55b3e as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/A/cf65c7a4c4994b898e5df62963e55b3e 2024-11-23T16:14:42,081 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/A/cf65c7a4c4994b898e5df62963e55b3e, entries=150, sequenceid=47, filesize=11.7 K 2024-11-23T16:14:42,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/.tmp/B/fbb105af53f64607977e5eb3818f7143 as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/B/fbb105af53f64607977e5eb3818f7143 2024-11-23T16:14:42,087 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/B/fbb105af53f64607977e5eb3818f7143, entries=150, sequenceid=47, filesize=11.7 K 2024-11-23T16:14:42,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/.tmp/C/22c41e52183c4f37949452f6e1234d84 as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/C/22c41e52183c4f37949452f6e1234d84 2024-11-23T16:14:42,093 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/C/22c41e52183c4f37949452f6e1234d84, entries=150, sequenceid=47, filesize=11.7 K 2024-11-23T16:14:42,093 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(3140): Finished flush of dataSize ~129.75 KB/132866, heapSize ~365.51 KB/374280, currentSize=53.67 KB/54960 for 5b34e731541fe76c476f23c5dc5aa28e in 486ms, sequenceid=47, compaction requested=false 2024-11-23T16:14:42,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2603): Flush status journal for 5b34e731541fe76c476f23c5dc5aa28e: 2024-11-23T16:14:42,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e. 2024-11-23T16:14:42,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=63 2024-11-23T16:14:42,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.HMaster(4169): Remote procedure done, pid=63 2024-11-23T16:14:42,097 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=63, resume processing ppid=62 2024-11-23T16:14:42,097 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, ppid=62, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5710 sec 2024-11-23T16:14:42,099 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees in 1.5770 sec 2024-11-23T16:14:42,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(8855): Flush requested on 5b34e731541fe76c476f23c5dc5aa28e 2024-11-23T16:14:42,120 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5b34e731541fe76c476f23c5dc5aa28e 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-23T16:14:42,120 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5b34e731541fe76c476f23c5dc5aa28e, store=A 2024-11-23T16:14:42,120 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-23T16:14:42,120 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5b34e731541fe76c476f23c5dc5aa28e, store=B 2024-11-23T16:14:42,120 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-23T16:14:42,120 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5b34e731541fe76c476f23c5dc5aa28e, store=C 2024-11-23T16:14:42,120 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-23T16:14:42,138 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/.tmp/A/6c8c761768e0445cb6186523cccc5209 is 50, key is test_row_0/A:col10/1732378482118/Put/seqid=0 2024-11-23T16:14:42,171 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742068_1244 (size=12001) 2024-11-23T16:14:42,171 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/.tmp/A/6c8c761768e0445cb6186523cccc5209 2024-11-23T16:14:42,175 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:42,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50424 deadline: 1732378542171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:42,176 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:42,176 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:42,176 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:14:42,178 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:42,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50480 deadline: 1732378542173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:42,178 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:42,178 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:42,178 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:14:42,184 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/.tmp/B/f3c03781921d4b4eb47abceedfcbf825 is 50, key is test_row_0/B:col10/1732378482118/Put/seqid=0 2024-11-23T16:14:42,199 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742069_1245 (size=12001) 2024-11-23T16:14:42,281 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:42,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50424 deadline: 1732378542278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:42,281 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:42,281 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:42,281 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:14:42,290 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:42,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50480 deadline: 1732378542287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:42,291 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:42,291 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:42,291 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:14:42,491 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:42,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50424 deadline: 1732378542488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:42,492 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:42,492 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:42,492 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:14:42,503 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:42,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50480 deadline: 1732378542498, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:42,503 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:42,503 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:42,503 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:14:42,600 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/.tmp/B/f3c03781921d4b4eb47abceedfcbf825 2024-11-23T16:14:42,608 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/.tmp/C/e131724238234ebca997fd18293eabfe is 50, key is test_row_0/C:col10/1732378482118/Put/seqid=0 2024-11-23T16:14:42,613 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742070_1246 (size=12001) 2024-11-23T16:14:42,614 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/.tmp/C/e131724238234ebca997fd18293eabfe 2024-11-23T16:14:42,620 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/.tmp/A/6c8c761768e0445cb6186523cccc5209 as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/A/6c8c761768e0445cb6186523cccc5209 2024-11-23T16:14:42,624 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/A/6c8c761768e0445cb6186523cccc5209, entries=150, sequenceid=59, filesize=11.7 K 2024-11-23T16:14:42,625 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/.tmp/B/f3c03781921d4b4eb47abceedfcbf825 as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/B/f3c03781921d4b4eb47abceedfcbf825 2024-11-23T16:14:42,629 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/B/f3c03781921d4b4eb47abceedfcbf825, entries=150, sequenceid=59, filesize=11.7 K 2024-11-23T16:14:42,630 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/.tmp/C/e131724238234ebca997fd18293eabfe as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/C/e131724238234ebca997fd18293eabfe 2024-11-23T16:14:42,635 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/C/e131724238234ebca997fd18293eabfe, entries=150, sequenceid=59, filesize=11.7 K 2024-11-23T16:14:42,636 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 5b34e731541fe76c476f23c5dc5aa28e in 515ms, sequenceid=59, compaction requested=true 2024-11-23T16:14:42,636 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5b34e731541fe76c476f23c5dc5aa28e: 2024-11-23T16:14:42,636 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5b34e731541fe76c476f23c5dc5aa28e:A, priority=-2147483648, current under compaction store size is 1 2024-11-23T16:14:42,636 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T16:14:42,636 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5b34e731541fe76c476f23c5dc5aa28e:B, priority=-2147483648, current under compaction store size is 2 2024-11-23T16:14:42,636 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T16:14:42,636 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5b34e731541fe76c476f23c5dc5aa28e:C, priority=-2147483648, current under compaction store size is 3 2024-11-23T16:14:42,636 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T16:14:42,636 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T16:14:42,636 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T16:14:42,637 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T16:14:42,637 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T16:14:42,637 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HStore(1541): 5b34e731541fe76c476f23c5dc5aa28e/A is initiating minor compaction (all files) 2024-11-23T16:14:42,637 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HStore(1541): 5b34e731541fe76c476f23c5dc5aa28e/B is initiating minor compaction (all files) 2024-11-23T16:14:42,637 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5b34e731541fe76c476f23c5dc5aa28e/A in TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e. 2024-11-23T16:14:42,637 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5b34e731541fe76c476f23c5dc5aa28e/B in TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e. 2024-11-23T16:14:42,637 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/A/cc582adb0e4a4cc6b18911da9d4817aa, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/A/cf65c7a4c4994b898e5df62963e55b3e, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/A/6c8c761768e0445cb6186523cccc5209] into tmpdir=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/.tmp, totalSize=35.2 K 2024-11-23T16:14:42,637 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/B/09a9483ab01c4a93b5dd72f29e8976fe, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/B/fbb105af53f64607977e5eb3818f7143, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/B/f3c03781921d4b4eb47abceedfcbf825] into tmpdir=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/.tmp, totalSize=35.2 K 2024-11-23T16:14:42,638 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.Compactor(225): Compacting 09a9483ab01c4a93b5dd72f29e8976fe, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1732378480536 2024-11-23T16:14:42,638 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.Compactor(225): Compacting cc582adb0e4a4cc6b18911da9d4817aa, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1732378480536 2024-11-23T16:14:42,638 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.Compactor(225): Compacting fbb105af53f64607977e5eb3818f7143, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1732378480665 2024-11-23T16:14:42,639 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.Compactor(225): Compacting cf65c7a4c4994b898e5df62963e55b3e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1732378480665 2024-11-23T16:14:42,639 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.Compactor(225): Compacting f3c03781921d4b4eb47abceedfcbf825, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1732378481797 2024-11-23T16:14:42,639 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6c8c761768e0445cb6186523cccc5209, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1732378481797 2024-11-23T16:14:42,654 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5b34e731541fe76c476f23c5dc5aa28e#A#compaction#201 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-23T16:14:42,655 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/.tmp/A/6098615294af43fb8b64b558f9dec825 is 50, key is test_row_0/A:col10/1732378482118/Put/seqid=0 2024-11-23T16:14:42,656 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5b34e731541fe76c476f23c5dc5aa28e#B#compaction#202 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T16:14:42,656 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/.tmp/B/beeba66974a54d2cb38347f72f7c20a9 is 50, key is test_row_0/B:col10/1732378482118/Put/seqid=0 2024-11-23T16:14:42,673 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742071_1247 (size=12104) 2024-11-23T16:14:42,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-23T16:14:42,679 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-23T16:14:42,681 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-23T16:14:42,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] procedure2.ProcedureExecutor(1139): Stored pid=64, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees 2024-11-23T16:14:42,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-23T16:14:42,684 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=64, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-23T16:14:42,686 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=64, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-23T16:14:42,686 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=65, ppid=64, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-23T16:14:42,691 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/.tmp/A/6098615294af43fb8b64b558f9dec825 as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/A/6098615294af43fb8b64b558f9dec825 2024-11-23T16:14:42,700 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5b34e731541fe76c476f23c5dc5aa28e/A of 5b34e731541fe76c476f23c5dc5aa28e into 6098615294af43fb8b64b558f9dec825(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T16:14:42,700 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5b34e731541fe76c476f23c5dc5aa28e: 2024-11-23T16:14:42,700 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., storeName=5b34e731541fe76c476f23c5dc5aa28e/A, priority=13, startTime=1732378482636; duration=0sec 2024-11-23T16:14:42,700 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T16:14:42,700 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5b34e731541fe76c476f23c5dc5aa28e:A 2024-11-23T16:14:42,700 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T16:14:42,702 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T16:14:42,702 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HStore(1541): 5b34e731541fe76c476f23c5dc5aa28e/C is initiating minor compaction (all files) 2024-11-23T16:14:42,702 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5b34e731541fe76c476f23c5dc5aa28e/C in TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e. 2024-11-23T16:14:42,702 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/C/011c9d7b98614d99b81fab8639fdf62d, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/C/22c41e52183c4f37949452f6e1234d84, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/C/e131724238234ebca997fd18293eabfe] into tmpdir=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/.tmp, totalSize=35.2 K 2024-11-23T16:14:42,702 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.Compactor(225): Compacting 011c9d7b98614d99b81fab8639fdf62d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1732378480536 2024-11-23T16:14:42,703 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.Compactor(225): Compacting 22c41e52183c4f37949452f6e1234d84, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1732378480665 2024-11-23T16:14:42,703 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.Compactor(225): Compacting e131724238234ebca997fd18293eabfe, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1732378481797 2024-11-23T16:14:42,706 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742072_1248 (size=12104) 2024-11-23T16:14:42,721 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/.tmp/B/beeba66974a54d2cb38347f72f7c20a9 as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/B/beeba66974a54d2cb38347f72f7c20a9 2024-11-23T16:14:42,728 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5b34e731541fe76c476f23c5dc5aa28e#C#compaction#203 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T16:14:42,729 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/.tmp/C/cb95b6a55dd14f97986641f953711dc6 is 50, key is test_row_0/C:col10/1732378482118/Put/seqid=0 2024-11-23T16:14:42,731 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5b34e731541fe76c476f23c5dc5aa28e/B of 5b34e731541fe76c476f23c5dc5aa28e into beeba66974a54d2cb38347f72f7c20a9(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T16:14:42,731 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5b34e731541fe76c476f23c5dc5aa28e: 2024-11-23T16:14:42,731 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., storeName=5b34e731541fe76c476f23c5dc5aa28e/B, priority=13, startTime=1732378482636; duration=0sec 2024-11-23T16:14:42,731 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T16:14:42,731 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5b34e731541fe76c476f23c5dc5aa28e:B 2024-11-23T16:14:42,764 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742073_1249 (size=12104) 2024-11-23T16:14:42,774 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/.tmp/C/cb95b6a55dd14f97986641f953711dc6 as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/C/cb95b6a55dd14f97986641f953711dc6 2024-11-23T16:14:42,781 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5b34e731541fe76c476f23c5dc5aa28e/C of 5b34e731541fe76c476f23c5dc5aa28e into cb95b6a55dd14f97986641f953711dc6(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T16:14:42,781 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5b34e731541fe76c476f23c5dc5aa28e: 2024-11-23T16:14:42,781 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., storeName=5b34e731541fe76c476f23c5dc5aa28e/C, priority=13, startTime=1732378482636; duration=0sec 2024-11-23T16:14:42,781 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T16:14:42,781 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5b34e731541fe76c476f23c5dc5aa28e:C 2024-11-23T16:14:42,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-23T16:14:42,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(8855): Flush requested on 5b34e731541fe76c476f23c5dc5aa28e 2024-11-23T16:14:42,802 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5b34e731541fe76c476f23c5dc5aa28e 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-11-23T16:14:42,802 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5b34e731541fe76c476f23c5dc5aa28e, store=A 2024-11-23T16:14:42,803 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-23T16:14:42,803 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5b34e731541fe76c476f23c5dc5aa28e, store=B 2024-11-23T16:14:42,803 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-23T16:14:42,803 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5b34e731541fe76c476f23c5dc5aa28e, store=C 2024-11-23T16:14:42,803 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-23T16:14:42,811 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/.tmp/A/95d5b137d6e14adf80aa0c4f06f38564 is 50, key is test_row_0/A:col10/1732378482801/Put/seqid=0 2024-11-23T16:14:42,830 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:42,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50480 deadline: 1732378542825, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:42,830 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:42,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50454 deadline: 1732378542828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:42,831 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:42,831 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:42,831 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 1a7c5470b472,44715,1732378420375 for 'test_row_0' in 5b34e731541fe76c476f23c5dc5aa28e of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2156 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-23T16:14:42,832 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:14:42,832 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:42,832 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:42,832 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:14:42,832 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:42,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50424 deadline: 1732378542829, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:42,833 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:42,833 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:42,833 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:14:42,839 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-23T16:14:42,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e. 2024-11-23T16:14:42,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e. as already flushing 2024-11-23T16:14:42,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e. 2024-11-23T16:14:42,840 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 5b34e731541fe76c476f23c5dc5aa28e, NAME => 'TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:14:42,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 5b34e731541fe76c476f23c5dc5aa28e, NAME => 'TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:14:42,840 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:42,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50486 deadline: 1732378542838, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:42,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5b34e731541fe76c476f23c5dc5aa28e, NAME => 'TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5b34e731541fe76c476f23c5dc5aa28e, NAME => 'TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:14:42,841 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 1a7c5470b472,44715,1732378420375 for 'test_row_2' in 5b34e731541fe76c476f23c5dc5aa28e of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2166 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-23T16:14:42,842 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:42,842 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:42,842 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:14:42,850 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:42,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50462 deadline: 1732378542848, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:42,851 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 1a7c5470b472,44715,1732378420375 for 'test_row_2' in 5b34e731541fe76c476f23c5dc5aa28e of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2168 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-23T16:14:42,852 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:42,852 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:42,852 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:14:42,877 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742074_1250 (size=12001) 2024-11-23T16:14:42,941 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:42,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50480 deadline: 1732378542939, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:42,942 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:42,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50424 deadline: 1732378542939, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:42,942 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:42,942 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:42,942 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:14:42,942 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:42,942 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:42,942 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:14:42,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-23T16:14:43,000 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-23T16:14:43,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e. 2024-11-23T16:14:43,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e. as already flushing 2024-11-23T16:14:43,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e. 2024-11-23T16:14:43,000 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 5b34e731541fe76c476f23c5dc5aa28e, NAME => 'TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:14:43,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 5b34e731541fe76c476f23c5dc5aa28e, NAME => 'TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:14:43,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5b34e731541fe76c476f23c5dc5aa28e, NAME => 'TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5b34e731541fe76c476f23c5dc5aa28e, NAME => 'TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:14:43,153 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:43,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50424 deadline: 1732378543148, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:43,153 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:43,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50480 deadline: 1732378543148, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:43,154 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-23T16:14:43,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e. 2024-11-23T16:14:43,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e. as already flushing 2024-11-23T16:14:43,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e. 2024-11-23T16:14:43,154 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 5b34e731541fe76c476f23c5dc5aa28e, NAME => 'TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:14:43,154 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:43,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 5b34e731541fe76c476f23c5dc5aa28e, NAME => 'TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:14:43,154 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:43,154 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:14:43,155 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:43,155 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:43,155 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:14:43,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5b34e731541fe76c476f23c5dc5aa28e, NAME => 'TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5b34e731541fe76c476f23c5dc5aa28e, NAME => 'TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:14:43,278 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=83 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/.tmp/A/95d5b137d6e14adf80aa0c4f06f38564 2024-11-23T16:14:43,299 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/.tmp/B/a5c2b481b5434f13aa1931b71ef76079 is 50, key is test_row_0/B:col10/1732378482801/Put/seqid=0 2024-11-23T16:14:43,307 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-23T16:14:43,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e. 2024-11-23T16:14:43,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-23T16:14:43,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e. as already flushing 2024-11-23T16:14:43,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e. 2024-11-23T16:14:43,308 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 5b34e731541fe76c476f23c5dc5aa28e, NAME => 'TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:14:43,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 5b34e731541fe76c476f23c5dc5aa28e, NAME => 'TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:14:43,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5b34e731541fe76c476f23c5dc5aa28e, NAME => 'TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5b34e731541fe76c476f23c5dc5aa28e, NAME => 'TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:14:43,331 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742075_1251 (size=12001) 2024-11-23T16:14:43,459 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:43,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50424 deadline: 1732378543458, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:43,460 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:43,460 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:43,460 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:14:43,462 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-23T16:14:43,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e. 2024-11-23T16:14:43,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e. as already flushing 2024-11-23T16:14:43,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e. 2024-11-23T16:14:43,463 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 5b34e731541fe76c476f23c5dc5aa28e, NAME => 'TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:14:43,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 5b34e731541fe76c476f23c5dc5aa28e, NAME => 'TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:14:43,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5b34e731541fe76c476f23c5dc5aa28e, NAME => 'TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5b34e731541fe76c476f23c5dc5aa28e, NAME => 'TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:14:43,470 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:43,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50480 deadline: 1732378543468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:43,471 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:43,471 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:43,471 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:14:43,562 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-23T16:14:43,616 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-23T16:14:43,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e. 2024-11-23T16:14:43,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e. as already flushing 2024-11-23T16:14:43,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e. 2024-11-23T16:14:43,617 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 5b34e731541fe76c476f23c5dc5aa28e, NAME => 'TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:14:43,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 5b34e731541fe76c476f23c5dc5aa28e, NAME => 'TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:14:43,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5b34e731541fe76c476f23c5dc5aa28e, NAME => 'TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5b34e731541fe76c476f23c5dc5aa28e, NAME => 'TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:14:43,735 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=83 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/.tmp/B/a5c2b481b5434f13aa1931b71ef76079 2024-11-23T16:14:43,752 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/.tmp/C/0ec7a896d69c4ca1a2ab2571709796b6 is 50, key is test_row_0/C:col10/1732378482801/Put/seqid=0 2024-11-23T16:14:43,773 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-23T16:14:43,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e. 2024-11-23T16:14:43,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e. as already flushing 2024-11-23T16:14:43,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e. 2024-11-23T16:14:43,774 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 5b34e731541fe76c476f23c5dc5aa28e, NAME => 'TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:14:43,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 5b34e731541fe76c476f23c5dc5aa28e, NAME => 'TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:14:43,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5b34e731541fe76c476f23c5dc5aa28e, NAME => 'TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5b34e731541fe76c476f23c5dc5aa28e, NAME => 'TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:14:43,793 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742076_1252 (size=12001) 2024-11-23T16:14:43,794 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=83 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/.tmp/C/0ec7a896d69c4ca1a2ab2571709796b6 2024-11-23T16:14:43,800 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/.tmp/A/95d5b137d6e14adf80aa0c4f06f38564 as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/A/95d5b137d6e14adf80aa0c4f06f38564 2024-11-23T16:14:43,809 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/A/95d5b137d6e14adf80aa0c4f06f38564, entries=150, sequenceid=83, filesize=11.7 K 2024-11-23T16:14:43,811 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/.tmp/B/a5c2b481b5434f13aa1931b71ef76079 as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/B/a5c2b481b5434f13aa1931b71ef76079 2024-11-23T16:14:43,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-23T16:14:43,820 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/B/a5c2b481b5434f13aa1931b71ef76079, entries=150, sequenceid=83, filesize=11.7 K 2024-11-23T16:14:43,822 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/.tmp/C/0ec7a896d69c4ca1a2ab2571709796b6 as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/C/0ec7a896d69c4ca1a2ab2571709796b6 2024-11-23T16:14:43,832 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/C/0ec7a896d69c4ca1a2ab2571709796b6, entries=150, sequenceid=83, filesize=11.7 K 2024-11-23T16:14:43,833 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=53.67 KB/54960 for 5b34e731541fe76c476f23c5dc5aa28e in 1031ms, sequenceid=83, compaction requested=false 2024-11-23T16:14:43,833 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5b34e731541fe76c476f23c5dc5aa28e: 2024-11-23T16:14:43,927 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-23T16:14:43,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e. 2024-11-23T16:14:43,928 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2902): Flushing 5b34e731541fe76c476f23c5dc5aa28e 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-23T16:14:43,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5b34e731541fe76c476f23c5dc5aa28e, store=A 2024-11-23T16:14:43,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-23T16:14:43,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5b34e731541fe76c476f23c5dc5aa28e, store=B 2024-11-23T16:14:43,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-23T16:14:43,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5b34e731541fe76c476f23c5dc5aa28e, store=C 2024-11-23T16:14:43,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-23T16:14:43,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/.tmp/A/bf74df63ad144d31ac9ea6914ff95f03 is 50, key is test_row_0/A:col10/1732378482828/Put/seqid=0 2024-11-23T16:14:43,967 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742077_1253 (size=12001) 2024-11-23T16:14:43,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(8855): Flush requested on 5b34e731541fe76c476f23c5dc5aa28e 2024-11-23T16:14:43,970 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e. as already flushing 2024-11-23T16:14:44,029 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1a7c5470b472:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732378484010, max timestamp=1732378484024Num uniques 859; 2024-11-23T16:14:44,029 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1a7c5470b472:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-23T16:14:44,029 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1a7c5470b472:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732378484010, max timestamp=1732378484024Num uniques 859; 2024-11-23T16:14:44,029 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1a7c5470b472:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732378484010, max timestamp=1732378484024Num uniques 859; 2024-11-23T16:14:44,029 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1a7c5470b472:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-23T16:14:44,029 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1a7c5470b472:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-23T16:14:44,090 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:44,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50424 deadline: 1732378544084, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:44,091 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:44,091 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:44,091 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:14:44,091 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:44,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50480 deadline: 1732378544086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:44,092 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:44,092 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:44,092 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:14:44,199 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:44,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50424 deadline: 1732378544198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:44,200 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:44,200 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:44,200 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:44,200 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:14:44,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50480 deadline: 1732378544198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:44,200 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:44,200 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:44,200 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:14:44,368 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=94 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/.tmp/A/bf74df63ad144d31ac9ea6914ff95f03 2024-11-23T16:14:44,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/.tmp/B/7516c502ee024ee0b7fee15538b1d040 is 50, key is test_row_0/B:col10/1732378482828/Put/seqid=0 2024-11-23T16:14:44,412 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:44,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50424 deadline: 1732378544409, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:44,412 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:44,412 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:44,413 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:14:44,413 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:44,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50480 deadline: 1732378544409, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:44,413 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:44,413 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:44,413 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:14:44,418 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742078_1254 (size=12001) 2024-11-23T16:14:44,722 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:44,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50480 deadline: 1732378544718, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:44,722 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:44,722 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:44,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50424 deadline: 1732378544718, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:44,722 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:44,722 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:14:44,723 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:44,723 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:44,723 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:14:44,819 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=94 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/.tmp/B/7516c502ee024ee0b7fee15538b1d040 2024-11-23T16:14:44,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/.tmp/C/6aa31c68f85f450891c718706a92e59c is 50, key is test_row_0/C:col10/1732378482828/Put/seqid=0 2024-11-23T16:14:44,833 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742079_1255 (size=12001) 2024-11-23T16:14:44,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-23T16:14:44,841 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:44,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50454 deadline: 1732378544838, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:44,842 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 1a7c5470b472,44715,1732378420375 for 'test_row_0' in 5b34e731541fe76c476f23c5dc5aa28e of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4167 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-23T16:14:44,842 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:44,842 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:44,842 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:14:44,871 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:44,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50486 deadline: 1732378544868, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:44,872 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 1a7c5470b472,44715,1732378420375 for 'test_row_2' in 5b34e731541fe76c476f23c5dc5aa28e of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4197 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-23T16:14:44,873 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:44,873 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:44,873 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:14:44,878 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:44,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50462 deadline: 1732378544878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:44,879 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 1a7c5470b472,44715,1732378420375 for 'test_row_2' in 5b34e731541fe76c476f23c5dc5aa28e of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4195 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-23T16:14:44,879 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:44,879 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:44,879 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:14:45,231 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:45,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50480 deadline: 1732378545228, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:45,232 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:45,232 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:45,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50424 deadline: 1732378545228, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:45,232 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:45,232 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:14:45,233 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:45,233 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:45,233 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:14:45,233 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=94 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/.tmp/C/6aa31c68f85f450891c718706a92e59c 2024-11-23T16:14:45,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/.tmp/A/bf74df63ad144d31ac9ea6914ff95f03 as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/A/bf74df63ad144d31ac9ea6914ff95f03 2024-11-23T16:14:45,256 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/A/bf74df63ad144d31ac9ea6914ff95f03, entries=150, sequenceid=94, filesize=11.7 K 2024-11-23T16:14:45,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/.tmp/B/7516c502ee024ee0b7fee15538b1d040 as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/B/7516c502ee024ee0b7fee15538b1d040 2024-11-23T16:14:45,262 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/B/7516c502ee024ee0b7fee15538b1d040, entries=150, sequenceid=94, filesize=11.7 K 2024-11-23T16:14:45,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/.tmp/C/6aa31c68f85f450891c718706a92e59c as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/C/6aa31c68f85f450891c718706a92e59c 2024-11-23T16:14:45,267 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/C/6aa31c68f85f450891c718706a92e59c, entries=150, sequenceid=94, filesize=11.7 K 2024-11-23T16:14:45,268 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 5b34e731541fe76c476f23c5dc5aa28e in 1340ms, sequenceid=94, compaction requested=true 2024-11-23T16:14:45,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2603): Flush status journal for 5b34e731541fe76c476f23c5dc5aa28e: 2024-11-23T16:14:45,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e. 2024-11-23T16:14:45,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=65 2024-11-23T16:14:45,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.HMaster(4169): Remote procedure done, pid=65 2024-11-23T16:14:45,272 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=65, resume processing ppid=64 2024-11-23T16:14:45,273 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=65, ppid=64, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.5840 sec 2024-11-23T16:14:45,275 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=64, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees in 2.5920 sec 2024-11-23T16:14:46,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(8855): Flush requested on 5b34e731541fe76c476f23c5dc5aa28e 2024-11-23T16:14:46,242 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5b34e731541fe76c476f23c5dc5aa28e 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-11-23T16:14:46,242 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5b34e731541fe76c476f23c5dc5aa28e, store=A 2024-11-23T16:14:46,243 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-23T16:14:46,243 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5b34e731541fe76c476f23c5dc5aa28e, store=B 2024-11-23T16:14:46,243 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-23T16:14:46,243 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5b34e731541fe76c476f23c5dc5aa28e, store=C 2024-11-23T16:14:46,243 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-23T16:14:46,263 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/.tmp/A/f56d4b34d42449eba8c6c7be0ab73aa0 is 50, key is test_row_0/A:col10/1732378486240/Put/seqid=0 2024-11-23T16:14:46,264 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:46,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50480 deadline: 1732378546260, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:46,264 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:46,265 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:46,265 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:14:46,267 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:46,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50424 deadline: 1732378546262, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:46,268 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742080_1256 (size=14441) 2024-11-23T16:14:46,269 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:46,269 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:46,269 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:14:46,269 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=131 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/.tmp/A/f56d4b34d42449eba8c6c7be0ab73aa0 2024-11-23T16:14:46,281 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/.tmp/B/a1359f923efa46d196a5e465dd7a1df1 is 50, key is test_row_0/B:col10/1732378486240/Put/seqid=0 2024-11-23T16:14:46,295 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742081_1257 (size=12101) 2024-11-23T16:14:46,300 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=131 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/.tmp/B/a1359f923efa46d196a5e465dd7a1df1 2024-11-23T16:14:46,311 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/.tmp/C/ff1f7c573de546d893d1db073b59e412 is 50, key is test_row_0/C:col10/1732378486240/Put/seqid=0 2024-11-23T16:14:46,324 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742082_1258 (size=12101) 2024-11-23T16:14:46,373 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:46,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50480 deadline: 1732378546368, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:46,373 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:46,373 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:46,373 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:14:46,382 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:46,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50424 deadline: 1732378546378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:46,382 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:46,382 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:46,382 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:14:46,579 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:46,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50480 deadline: 1732378546578, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:46,579 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:46,579 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:46,579 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:14:46,591 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:46,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50424 deadline: 1732378546587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:46,591 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:46,592 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:46,592 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:14:46,725 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=131 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/.tmp/C/ff1f7c573de546d893d1db073b59e412 2024-11-23T16:14:46,734 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/.tmp/A/f56d4b34d42449eba8c6c7be0ab73aa0 as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/A/f56d4b34d42449eba8c6c7be0ab73aa0 2024-11-23T16:14:46,741 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/A/f56d4b34d42449eba8c6c7be0ab73aa0, entries=200, sequenceid=131, filesize=14.1 K 2024-11-23T16:14:46,743 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/.tmp/B/a1359f923efa46d196a5e465dd7a1df1 as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/B/a1359f923efa46d196a5e465dd7a1df1 2024-11-23T16:14:46,747 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/B/a1359f923efa46d196a5e465dd7a1df1, entries=150, sequenceid=131, filesize=11.8 K 2024-11-23T16:14:46,748 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/.tmp/C/ff1f7c573de546d893d1db073b59e412 as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/C/ff1f7c573de546d893d1db073b59e412 2024-11-23T16:14:46,755 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/C/ff1f7c573de546d893d1db073b59e412, entries=150, sequenceid=131, filesize=11.8 K 2024-11-23T16:14:46,756 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=46.96 KB/48090 for 5b34e731541fe76c476f23c5dc5aa28e in 514ms, sequenceid=131, compaction requested=true 2024-11-23T16:14:46,756 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5b34e731541fe76c476f23c5dc5aa28e: 2024-11-23T16:14:46,757 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5b34e731541fe76c476f23c5dc5aa28e:A, priority=-2147483648, current under compaction store size is 1 2024-11-23T16:14:46,757 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T16:14:46,757 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-23T16:14:46,757 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5b34e731541fe76c476f23c5dc5aa28e:B, priority=-2147483648, current under compaction store size is 2 2024-11-23T16:14:46,757 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T16:14:46,757 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5b34e731541fe76c476f23c5dc5aa28e:C, priority=-2147483648, current under compaction store size is 3 2024-11-23T16:14:46,757 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-23T16:14:46,757 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T16:14:46,758 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50547 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-23T16:14:46,758 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HStore(1541): 5b34e731541fe76c476f23c5dc5aa28e/A is initiating minor compaction (all files) 2024-11-23T16:14:46,758 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5b34e731541fe76c476f23c5dc5aa28e/A in TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e. 2024-11-23T16:14:46,758 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/A/6098615294af43fb8b64b558f9dec825, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/A/95d5b137d6e14adf80aa0c4f06f38564, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/A/bf74df63ad144d31ac9ea6914ff95f03, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/A/f56d4b34d42449eba8c6c7be0ab73aa0] into tmpdir=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/.tmp, totalSize=49.4 K 2024-11-23T16:14:46,759 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48207 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-23T16:14:46,759 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6098615294af43fb8b64b558f9dec825, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1732378481797 2024-11-23T16:14:46,759 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HStore(1541): 5b34e731541fe76c476f23c5dc5aa28e/B is initiating minor compaction (all files) 2024-11-23T16:14:46,759 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5b34e731541fe76c476f23c5dc5aa28e/B in TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e. 2024-11-23T16:14:46,759 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/B/beeba66974a54d2cb38347f72f7c20a9, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/B/a5c2b481b5434f13aa1931b71ef76079, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/B/7516c502ee024ee0b7fee15538b1d040, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/B/a1359f923efa46d196a5e465dd7a1df1] into tmpdir=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/.tmp, totalSize=47.1 K 2024-11-23T16:14:46,760 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.Compactor(225): Compacting beeba66974a54d2cb38347f72f7c20a9, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1732378481797 2024-11-23T16:14:46,760 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.Compactor(225): Compacting 95d5b137d6e14adf80aa0c4f06f38564, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1732378482167 2024-11-23T16:14:46,761 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.Compactor(225): Compacting bf74df63ad144d31ac9ea6914ff95f03, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1732378482810 2024-11-23T16:14:46,761 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.Compactor(225): Compacting a5c2b481b5434f13aa1931b71ef76079, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1732378482167 2024-11-23T16:14:46,762 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.Compactor(225): Compacting f56d4b34d42449eba8c6c7be0ab73aa0, keycount=200, bloomtype=ROW, size=14.1 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1732378484053 2024-11-23T16:14:46,762 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.Compactor(225): Compacting 7516c502ee024ee0b7fee15538b1d040, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1732378482810 2024-11-23T16:14:46,763 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.Compactor(225): Compacting a1359f923efa46d196a5e465dd7a1df1, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1732378484082 2024-11-23T16:14:46,780 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5b34e731541fe76c476f23c5dc5aa28e#A#compaction#214 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-23T16:14:46,780 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5b34e731541fe76c476f23c5dc5aa28e#B#compaction#213 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T16:14:46,781 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/.tmp/A/5384f824458046c6a6eed0566315ab50 is 50, key is test_row_0/A:col10/1732378486240/Put/seqid=0 2024-11-23T16:14:46,781 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/.tmp/B/c1896e0d029b4e55b9718a155f980ad2 is 50, key is test_row_0/B:col10/1732378486240/Put/seqid=0 2024-11-23T16:14:46,795 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742084_1260 (size=12341) 2024-11-23T16:14:46,803 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/.tmp/B/c1896e0d029b4e55b9718a155f980ad2 as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/B/c1896e0d029b4e55b9718a155f980ad2 2024-11-23T16:14:46,808 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742083_1259 (size=12341) 2024-11-23T16:14:46,812 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 5b34e731541fe76c476f23c5dc5aa28e/B of 5b34e731541fe76c476f23c5dc5aa28e into c1896e0d029b4e55b9718a155f980ad2(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T16:14:46,812 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5b34e731541fe76c476f23c5dc5aa28e: 2024-11-23T16:14:46,812 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., storeName=5b34e731541fe76c476f23c5dc5aa28e/B, priority=12, startTime=1732378486757; duration=0sec 2024-11-23T16:14:46,812 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T16:14:46,813 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5b34e731541fe76c476f23c5dc5aa28e:B 2024-11-23T16:14:46,813 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-23T16:14:46,816 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48207 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-23T16:14:46,816 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HStore(1541): 5b34e731541fe76c476f23c5dc5aa28e/C is initiating minor compaction (all files) 2024-11-23T16:14:46,817 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5b34e731541fe76c476f23c5dc5aa28e/C in TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e. 2024-11-23T16:14:46,817 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/C/cb95b6a55dd14f97986641f953711dc6, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/C/0ec7a896d69c4ca1a2ab2571709796b6, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/C/6aa31c68f85f450891c718706a92e59c, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/C/ff1f7c573de546d893d1db073b59e412] into tmpdir=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/.tmp, totalSize=47.1 K 2024-11-23T16:14:46,817 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/.tmp/A/5384f824458046c6a6eed0566315ab50 as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/A/5384f824458046c6a6eed0566315ab50 2024-11-23T16:14:46,817 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.Compactor(225): Compacting cb95b6a55dd14f97986641f953711dc6, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1732378481797 2024-11-23T16:14:46,818 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.Compactor(225): Compacting 0ec7a896d69c4ca1a2ab2571709796b6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1732378482167 2024-11-23T16:14:46,819 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.Compactor(225): Compacting 6aa31c68f85f450891c718706a92e59c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1732378482810 2024-11-23T16:14:46,819 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.Compactor(225): Compacting ff1f7c573de546d893d1db073b59e412, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1732378484082 2024-11-23T16:14:46,826 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 5b34e731541fe76c476f23c5dc5aa28e/A of 5b34e731541fe76c476f23c5dc5aa28e into 5384f824458046c6a6eed0566315ab50(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T16:14:46,826 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5b34e731541fe76c476f23c5dc5aa28e: 2024-11-23T16:14:46,826 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., storeName=5b34e731541fe76c476f23c5dc5aa28e/A, priority=12, startTime=1732378486757; duration=0sec 2024-11-23T16:14:46,827 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T16:14:46,827 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5b34e731541fe76c476f23c5dc5aa28e:A 2024-11-23T16:14:46,840 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5b34e731541fe76c476f23c5dc5aa28e#C#compaction#215 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T16:14:46,841 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/.tmp/C/367246ca68e04b1fb74befd857be030e is 50, key is test_row_0/C:col10/1732378486240/Put/seqid=0 2024-11-23T16:14:46,847 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742085_1261 (size=12341) 2024-11-23T16:14:46,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-23T16:14:46,849 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-23T16:14:46,849 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-23T16:14:46,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] procedure2.ProcedureExecutor(1139): Stored pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees 2024-11-23T16:14:46,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-23T16:14:46,852 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-23T16:14:46,853 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-23T16:14:46,853 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=67, ppid=66, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-23T16:14:46,858 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/.tmp/C/367246ca68e04b1fb74befd857be030e as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/C/367246ca68e04b1fb74befd857be030e 2024-11-23T16:14:46,866 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 5b34e731541fe76c476f23c5dc5aa28e/C of 5b34e731541fe76c476f23c5dc5aa28e into 367246ca68e04b1fb74befd857be030e(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T16:14:46,866 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5b34e731541fe76c476f23c5dc5aa28e: 2024-11-23T16:14:46,866 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., storeName=5b34e731541fe76c476f23c5dc5aa28e/C, priority=12, startTime=1732378486757; duration=0sec 2024-11-23T16:14:46,866 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T16:14:46,866 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5b34e731541fe76c476f23c5dc5aa28e:C 2024-11-23T16:14:46,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(8855): Flush requested on 5b34e731541fe76c476f23c5dc5aa28e 2024-11-23T16:14:46,889 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5b34e731541fe76c476f23c5dc5aa28e 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-23T16:14:46,890 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5b34e731541fe76c476f23c5dc5aa28e, store=A 2024-11-23T16:14:46,890 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-23T16:14:46,890 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5b34e731541fe76c476f23c5dc5aa28e, store=B 2024-11-23T16:14:46,890 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-23T16:14:46,890 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5b34e731541fe76c476f23c5dc5aa28e, store=C 2024-11-23T16:14:46,890 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-23T16:14:46,898 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/.tmp/A/3879c73ea540499487e8660ac615f037 is 50, key is test_row_0/A:col10/1732378486888/Put/seqid=0 2024-11-23T16:14:46,911 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742086_1262 (size=12151) 2024-11-23T16:14:46,913 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=145 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/.tmp/A/3879c73ea540499487e8660ac615f037 2024-11-23T16:14:46,926 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/.tmp/B/407996804dcb4a70841312a6eec170bd is 50, key is test_row_0/B:col10/1732378486888/Put/seqid=0 2024-11-23T16:14:46,944 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742087_1263 (size=12151) 2024-11-23T16:14:46,947 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1a7c5470b472:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732378486919, max timestamp=1732378486944Num uniques 859; 2024-11-23T16:14:46,948 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1a7c5470b472:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-23T16:14:46,948 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1a7c5470b472:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732378486919, max timestamp=1732378486944Num uniques 859; 2024-11-23T16:14:46,948 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1a7c5470b472:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-23T16:14:46,948 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1a7c5470b472:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732378486919, max timestamp=1732378486944Num uniques 859; 2024-11-23T16:14:46,948 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1a7c5470b472:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-23T16:14:46,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-23T16:14:47,005 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:47,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50424 deadline: 1732378546997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:47,005 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:47,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50480 deadline: 1732378546998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:47,006 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:47,006 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:47,006 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:47,006 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:14:47,006 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:47,006 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:14:47,008 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-23T16:14:47,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e. 2024-11-23T16:14:47,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e. as already flushing 2024-11-23T16:14:47,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e. 2024-11-23T16:14:47,008 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 5b34e731541fe76c476f23c5dc5aa28e, NAME => 'TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:14:47,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 5b34e731541fe76c476f23c5dc5aa28e, NAME => 'TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:14:47,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5b34e731541fe76c476f23c5dc5aa28e, NAME => 'TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5b34e731541fe76c476f23c5dc5aa28e, NAME => 'TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:14:47,109 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:47,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50480 deadline: 1732378547108, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:47,109 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:47,109 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:47,110 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:47,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50424 deadline: 1732378547108, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:47,110 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:14:47,110 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:47,110 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:47,110 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:14:47,162 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-23T16:14:47,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e. 2024-11-23T16:14:47,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e. as already flushing 2024-11-23T16:14:47,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e. 2024-11-23T16:14:47,163 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 5b34e731541fe76c476f23c5dc5aa28e, NAME => 'TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:14:47,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 5b34e731541fe76c476f23c5dc5aa28e, NAME => 'TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:14:47,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5b34e731541fe76c476f23c5dc5aa28e, NAME => 'TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5b34e731541fe76c476f23c5dc5aa28e, NAME => 'TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:14:47,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-23T16:14:47,322 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-23T16:14:47,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e. 2024-11-23T16:14:47,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e. as already flushing 2024-11-23T16:14:47,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e. 2024-11-23T16:14:47,323 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 5b34e731541fe76c476f23c5dc5aa28e, NAME => 'TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:14:47,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 5b34e731541fe76c476f23c5dc5aa28e, NAME => 'TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:14:47,323 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:47,323 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:47,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50480 deadline: 1732378547318, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:47,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50424 deadline: 1732378547318, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:47,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5b34e731541fe76c476f23c5dc5aa28e, NAME => 'TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5b34e731541fe76c476f23c5dc5aa28e, NAME => 'TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:14:47,324 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:47,324 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:47,324 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:47,324 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:47,324 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:14:47,324 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:14:47,346 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=145 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/.tmp/B/407996804dcb4a70841312a6eec170bd 2024-11-23T16:14:47,367 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/.tmp/C/a6bb103255e34ed98332935e32943a73 is 50, key is test_row_0/C:col10/1732378486888/Put/seqid=0 2024-11-23T16:14:47,386 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742088_1264 (size=12151) 2024-11-23T16:14:47,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-23T16:14:47,480 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-23T16:14:47,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e. 2024-11-23T16:14:47,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e. as already flushing 2024-11-23T16:14:47,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e. 2024-11-23T16:14:47,481 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 5b34e731541fe76c476f23c5dc5aa28e, NAME => 'TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:14:47,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 5b34e731541fe76c476f23c5dc5aa28e, NAME => 'TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:14:47,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5b34e731541fe76c476f23c5dc5aa28e, NAME => 'TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5b34e731541fe76c476f23c5dc5aa28e, NAME => 'TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:14:47,631 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:47,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50424 deadline: 1732378547628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:47,631 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:47,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50480 deadline: 1732378547628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:47,631 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:47,631 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:47,631 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:14:47,631 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:47,632 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:47,632 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:14:47,635 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-23T16:14:47,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e. 2024-11-23T16:14:47,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e. as already flushing 2024-11-23T16:14:47,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e. 2024-11-23T16:14:47,635 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 5b34e731541fe76c476f23c5dc5aa28e, NAME => 'TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:14:47,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 5b34e731541fe76c476f23c5dc5aa28e, NAME => 'TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:14:47,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5b34e731541fe76c476f23c5dc5aa28e, NAME => 'TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5b34e731541fe76c476f23c5dc5aa28e, NAME => 'TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:14:47,787 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=145 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/.tmp/C/a6bb103255e34ed98332935e32943a73 2024-11-23T16:14:47,788 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-23T16:14:47,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e. 2024-11-23T16:14:47,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e. as already flushing 2024-11-23T16:14:47,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e. 2024-11-23T16:14:47,788 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 5b34e731541fe76c476f23c5dc5aa28e, NAME => 'TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:14:47,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 5b34e731541fe76c476f23c5dc5aa28e, NAME => 'TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:14:47,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5b34e731541fe76c476f23c5dc5aa28e, NAME => 'TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5b34e731541fe76c476f23c5dc5aa28e, NAME => 'TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:14:47,793 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/.tmp/A/3879c73ea540499487e8660ac615f037 as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/A/3879c73ea540499487e8660ac615f037 2024-11-23T16:14:47,797 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/A/3879c73ea540499487e8660ac615f037, entries=150, sequenceid=145, filesize=11.9 K 2024-11-23T16:14:47,798 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/.tmp/B/407996804dcb4a70841312a6eec170bd as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/B/407996804dcb4a70841312a6eec170bd 2024-11-23T16:14:47,802 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/B/407996804dcb4a70841312a6eec170bd, entries=150, sequenceid=145, filesize=11.9 K 2024-11-23T16:14:47,803 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/.tmp/C/a6bb103255e34ed98332935e32943a73 as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/C/a6bb103255e34ed98332935e32943a73 2024-11-23T16:14:47,810 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/C/a6bb103255e34ed98332935e32943a73, entries=150, sequenceid=145, filesize=11.9 K 2024-11-23T16:14:47,811 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 5b34e731541fe76c476f23c5dc5aa28e in 922ms, sequenceid=145, compaction requested=false 2024-11-23T16:14:47,811 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5b34e731541fe76c476f23c5dc5aa28e: 2024-11-23T16:14:47,941 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-23T16:14:47,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e. 2024-11-23T16:14:47,942 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2902): Flushing 5b34e731541fe76c476f23c5dc5aa28e 3/3 column families, dataSize=127.47 KB heapSize=365.06 KB 2024-11-23T16:14:47,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5b34e731541fe76c476f23c5dc5aa28e, store=A 2024-11-23T16:14:47,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-23T16:14:47,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5b34e731541fe76c476f23c5dc5aa28e, store=B 2024-11-23T16:14:47,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-23T16:14:47,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5b34e731541fe76c476f23c5dc5aa28e, store=C 2024-11-23T16:14:47,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-23T16:14:47,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/.tmp/A/061f677f613e494fa09bff323886b641 is 50, key is test_row_0/A:col10/1732378486994/Put/seqid=0 2024-11-23T16:14:47,953 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742089_1265 (size=12151) 2024-11-23T16:14:47,959 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=181 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/.tmp/A/061f677f613e494fa09bff323886b641 2024-11-23T16:14:47,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/.tmp/B/4b1059e84c3c49acb94d51939ca6df50 is 50, key is test_row_0/B:col10/1732378486994/Put/seqid=0 2024-11-23T16:14:47,971 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742090_1266 (size=12151) 2024-11-23T16:14:47,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-23T16:14:48,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(8855): Flush requested on 5b34e731541fe76c476f23c5dc5aa28e 2024-11-23T16:14:48,141 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e. as already flushing 2024-11-23T16:14:48,163 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:48,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50480 deadline: 1732378548157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:48,163 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:48,163 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:48,163 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:48,163 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:14:48,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50424 deadline: 1732378548158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:48,163 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:48,163 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:48,163 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:14:48,269 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:48,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50480 deadline: 1732378548268, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:48,270 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:48,270 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:48,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50424 deadline: 1732378548268, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:48,270 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:48,270 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:14:48,270 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:48,270 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:48,270 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:14:48,372 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=181 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/.tmp/B/4b1059e84c3c49acb94d51939ca6df50 2024-11-23T16:14:48,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/.tmp/C/5e04e969a5e84e1da2019df4463e5336 is 50, key is test_row_0/C:col10/1732378486994/Put/seqid=0 2024-11-23T16:14:48,385 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742091_1267 (size=12151) 2024-11-23T16:14:48,482 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:48,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50480 deadline: 1732378548478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:48,482 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:48,482 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:48,482 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:14:48,482 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:48,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50424 deadline: 1732378548478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:48,483 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:48,483 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:48,483 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:14:48,785 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=181 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/.tmp/C/5e04e969a5e84e1da2019df4463e5336 2024-11-23T16:14:48,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/.tmp/A/061f677f613e494fa09bff323886b641 as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/A/061f677f613e494fa09bff323886b641 2024-11-23T16:14:48,792 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:48,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50480 deadline: 1732378548788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:48,792 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:48,792 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:48,792 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:48,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50424 deadline: 1732378548788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:48,792 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:14:48,793 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:48,793 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:48,793 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:14:48,794 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/A/061f677f613e494fa09bff323886b641, entries=150, sequenceid=181, filesize=11.9 K 2024-11-23T16:14:48,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/.tmp/B/4b1059e84c3c49acb94d51939ca6df50 as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/B/4b1059e84c3c49acb94d51939ca6df50 2024-11-23T16:14:48,799 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/B/4b1059e84c3c49acb94d51939ca6df50, entries=150, sequenceid=181, filesize=11.9 K 2024-11-23T16:14:48,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/.tmp/C/5e04e969a5e84e1da2019df4463e5336 as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/C/5e04e969a5e84e1da2019df4463e5336 2024-11-23T16:14:48,804 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/C/5e04e969a5e84e1da2019df4463e5336, entries=150, sequenceid=181, filesize=11.9 K 2024-11-23T16:14:48,805 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~365.02 KB/373776, currentSize=53.67 KB/54960 for 5b34e731541fe76c476f23c5dc5aa28e in 863ms, sequenceid=181, compaction requested=true 2024-11-23T16:14:48,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2603): Flush status journal for 5b34e731541fe76c476f23c5dc5aa28e: 2024-11-23T16:14:48,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e. 2024-11-23T16:14:48,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=67 2024-11-23T16:14:48,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.HMaster(4169): Remote procedure done, pid=67 2024-11-23T16:14:48,809 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=67, resume processing ppid=66 2024-11-23T16:14:48,809 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=67, ppid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9530 sec 2024-11-23T16:14:48,811 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees in 1.9610 sec 2024-11-23T16:14:48,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(8855): Flush requested on 5b34e731541fe76c476f23c5dc5aa28e 2024-11-23T16:14:48,881 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5b34e731541fe76c476f23c5dc5aa28e 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-23T16:14:48,881 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5b34e731541fe76c476f23c5dc5aa28e, store=A 2024-11-23T16:14:48,882 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-23T16:14:48,882 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5b34e731541fe76c476f23c5dc5aa28e, store=B 2024-11-23T16:14:48,882 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-23T16:14:48,882 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5b34e731541fe76c476f23c5dc5aa28e, store=C 2024-11-23T16:14:48,882 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-23T16:14:48,886 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/.tmp/A/46b14146244e4c5b85d34e3e83c18b9d is 50, key is test_row_0/A:col10/1732378488880/Put/seqid=0 2024-11-23T16:14:48,894 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742092_1268 (size=14541) 2024-11-23T16:14:48,934 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:48,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50454 deadline: 1732378548928, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:48,934 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:48,934 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:48,934 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:14:48,935 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:48,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50462 deadline: 1732378548930, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:48,935 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:48,935 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:48,935 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:48,935 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:14:48,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50486 deadline: 1732378548931, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:48,936 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:48,936 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:48,936 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:14:48,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-23T16:14:48,998 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-23T16:14:48,999 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-23T16:14:49,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] procedure2.ProcedureExecutor(1139): Stored pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees 2024-11-23T16:14:49,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-23T16:14:49,001 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-23T16:14:49,001 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-23T16:14:49,001 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=69, ppid=68, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-23T16:14:49,042 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:49,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50454 deadline: 1732378549037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:49,042 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:49,042 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:49,042 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:49,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50462 deadline: 1732378549038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:49,042 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:14:49,043 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:49,043 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:49,043 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:49,043 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:14:49,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50486 deadline: 1732378549038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:49,043 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:49,043 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:49,043 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:14:49,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-23T16:14:49,153 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-23T16:14:49,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e. 2024-11-23T16:14:49,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e. as already flushing 2024-11-23T16:14:49,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e. 2024-11-23T16:14:49,154 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 5b34e731541fe76c476f23c5dc5aa28e, NAME => 'TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:14:49,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 5b34e731541fe76c476f23c5dc5aa28e, NAME => 'TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:14:49,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5b34e731541fe76c476f23c5dc5aa28e, NAME => 'TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5b34e731541fe76c476f23c5dc5aa28e, NAME => 'TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:14:49,250 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:49,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50454 deadline: 1732378549247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:49,251 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:49,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50462 deadline: 1732378549247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:49,251 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:49,251 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:49,251 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:14:49,251 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:49,251 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:49,251 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:49,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50486 deadline: 1732378549248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:49,251 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:14:49,251 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:49,251 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:49,251 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:14:49,295 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=193 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/.tmp/A/46b14146244e4c5b85d34e3e83c18b9d 2024-11-23T16:14:49,301 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:49,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50424 deadline: 1732378549298, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:49,301 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:49,301 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:49,301 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:49,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50480 deadline: 1732378549298, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:49,301 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:14:49,301 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:49,301 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:49,302 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:14:49,302 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/.tmp/B/0027e9b63c7b41239c9f6219eac43145 is 50, key is test_row_0/B:col10/1732378488880/Put/seqid=0 2024-11-23T16:14:49,306 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742093_1269 (size=12151) 2024-11-23T16:14:49,307 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-23T16:14:49,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e. 2024-11-23T16:14:49,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e. as already flushing 2024-11-23T16:14:49,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e. 2024-11-23T16:14:49,307 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 5b34e731541fe76c476f23c5dc5aa28e, NAME => 'TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:14:49,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 5b34e731541fe76c476f23c5dc5aa28e, NAME => 'TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:14:49,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5b34e731541fe76c476f23c5dc5aa28e, NAME => 'TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5b34e731541fe76c476f23c5dc5aa28e, NAME => 'TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:14:49,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-23T16:14:49,460 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-23T16:14:49,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e. 2024-11-23T16:14:49,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e. as already flushing 2024-11-23T16:14:49,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e. 2024-11-23T16:14:49,460 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 5b34e731541fe76c476f23c5dc5aa28e, NAME => 'TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:14:49,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 5b34e731541fe76c476f23c5dc5aa28e, NAME => 'TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:14:49,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5b34e731541fe76c476f23c5dc5aa28e, NAME => 'TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5b34e731541fe76c476f23c5dc5aa28e, NAME => 'TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:14:49,560 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:49,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50454 deadline: 1732378549558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:49,561 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:49,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50462 deadline: 1732378549558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:49,561 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:49,561 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:49,561 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:14:49,561 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:49,561 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:49,561 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:14:49,561 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:49,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50486 deadline: 1732378549558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:49,561 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:49,561 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:49,562 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:14:49,614 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-23T16:14:49,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e. 2024-11-23T16:14:49,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e. as already flushing 2024-11-23T16:14:49,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e. 2024-11-23T16:14:49,614 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 5b34e731541fe76c476f23c5dc5aa28e, NAME => 'TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:14:49,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 5b34e731541fe76c476f23c5dc5aa28e, NAME => 'TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:14:49,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5b34e731541fe76c476f23c5dc5aa28e, NAME => 'TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5b34e731541fe76c476f23c5dc5aa28e, NAME => 'TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:14:49,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-23T16:14:49,706 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=193 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/.tmp/B/0027e9b63c7b41239c9f6219eac43145 2024-11-23T16:14:49,714 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/.tmp/C/5de3d97f873c4881b6a1898aff51ec7a is 50, key is test_row_0/C:col10/1732378488880/Put/seqid=0 2024-11-23T16:14:49,717 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742094_1270 (size=12151) 2024-11-23T16:14:49,767 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-23T16:14:49,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e. 2024-11-23T16:14:49,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e. as already flushing 2024-11-23T16:14:49,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e. 2024-11-23T16:14:49,767 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 5b34e731541fe76c476f23c5dc5aa28e, NAME => 'TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:14:49,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 5b34e731541fe76c476f23c5dc5aa28e, NAME => 'TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:14:49,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5b34e731541fe76c476f23c5dc5aa28e, NAME => 'TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5b34e731541fe76c476f23c5dc5aa28e, NAME => 'TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:14:49,920 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-23T16:14:49,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e. 2024-11-23T16:14:49,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e. as already flushing 2024-11-23T16:14:49,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e. 2024-11-23T16:14:49,920 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 5b34e731541fe76c476f23c5dc5aa28e, NAME => 'TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:14:49,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 5b34e731541fe76c476f23c5dc5aa28e, NAME => 'TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:14:49,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5b34e731541fe76c476f23c5dc5aa28e, NAME => 'TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5b34e731541fe76c476f23c5dc5aa28e, NAME => 'TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:14:50,071 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:50,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50454 deadline: 1732378550068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:50,072 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:50,072 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:50,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50462 deadline: 1732378550068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:50,072 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:50,072 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:14:50,072 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:50,072 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:50,072 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:50,072 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:14:50,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50486 deadline: 1732378550068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:50,072 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:50,073 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:50,073 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:14:50,073 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-23T16:14:50,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e. 2024-11-23T16:14:50,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e. as already flushing 2024-11-23T16:14:50,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e. 2024-11-23T16:14:50,073 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 5b34e731541fe76c476f23c5dc5aa28e, NAME => 'TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:14:50,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 5b34e731541fe76c476f23c5dc5aa28e, NAME => 'TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:14:50,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5b34e731541fe76c476f23c5dc5aa28e, NAME => 'TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5b34e731541fe76c476f23c5dc5aa28e, NAME => 'TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:14:50,118 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=193 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/.tmp/C/5de3d97f873c4881b6a1898aff51ec7a 2024-11-23T16:14:50,123 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/.tmp/A/46b14146244e4c5b85d34e3e83c18b9d as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/A/46b14146244e4c5b85d34e3e83c18b9d 2024-11-23T16:14:50,126 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/A/46b14146244e4c5b85d34e3e83c18b9d, entries=200, sequenceid=193, filesize=14.2 K 2024-11-23T16:14:50,127 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/.tmp/B/0027e9b63c7b41239c9f6219eac43145 as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/B/0027e9b63c7b41239c9f6219eac43145 2024-11-23T16:14:50,131 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/B/0027e9b63c7b41239c9f6219eac43145, entries=150, sequenceid=193, filesize=11.9 K 2024-11-23T16:14:50,132 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/.tmp/C/5de3d97f873c4881b6a1898aff51ec7a as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/C/5de3d97f873c4881b6a1898aff51ec7a 2024-11-23T16:14:50,136 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/C/5de3d97f873c4881b6a1898aff51ec7a, entries=150, sequenceid=193, filesize=11.9 K 2024-11-23T16:14:50,136 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 5b34e731541fe76c476f23c5dc5aa28e in 1255ms, sequenceid=193, compaction requested=true 2024-11-23T16:14:50,136 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5b34e731541fe76c476f23c5dc5aa28e: 2024-11-23T16:14:50,137 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5b34e731541fe76c476f23c5dc5aa28e:A, priority=-2147483648, current under compaction store size is 1 2024-11-23T16:14:50,137 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T16:14:50,137 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5b34e731541fe76c476f23c5dc5aa28e:B, priority=-2147483648, current under compaction store size is 2 2024-11-23T16:14:50,137 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T16:14:50,137 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5b34e731541fe76c476f23c5dc5aa28e:C, priority=-2147483648, current under compaction store size is 3 2024-11-23T16:14:50,137 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T16:14:50,137 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-23T16:14:50,137 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-23T16:14:50,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-23T16:14:50,138 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48794 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-23T16:14:50,138 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 51184 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-23T16:14:50,138 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HStore(1541): 5b34e731541fe76c476f23c5dc5aa28e/B is initiating minor compaction (all files) 2024-11-23T16:14:50,138 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HStore(1541): 5b34e731541fe76c476f23c5dc5aa28e/A is initiating minor compaction (all files) 2024-11-23T16:14:50,138 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5b34e731541fe76c476f23c5dc5aa28e/B in TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e. 2024-11-23T16:14:50,138 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5b34e731541fe76c476f23c5dc5aa28e/A in TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e. 2024-11-23T16:14:50,138 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/B/c1896e0d029b4e55b9718a155f980ad2, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/B/407996804dcb4a70841312a6eec170bd, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/B/4b1059e84c3c49acb94d51939ca6df50, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/B/0027e9b63c7b41239c9f6219eac43145] into tmpdir=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/.tmp, totalSize=47.7 K 2024-11-23T16:14:50,138 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/A/5384f824458046c6a6eed0566315ab50, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/A/3879c73ea540499487e8660ac615f037, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/A/061f677f613e494fa09bff323886b641, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/A/46b14146244e4c5b85d34e3e83c18b9d] into tmpdir=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/.tmp, totalSize=50.0 K 2024-11-23T16:14:50,139 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.Compactor(225): Compacting c1896e0d029b4e55b9718a155f980ad2, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1732378484082 2024-11-23T16:14:50,139 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5384f824458046c6a6eed0566315ab50, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1732378484082 2024-11-23T16:14:50,139 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.Compactor(225): Compacting 407996804dcb4a70841312a6eec170bd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1732378486258 2024-11-23T16:14:50,139 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3879c73ea540499487e8660ac615f037, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1732378486258 2024-11-23T16:14:50,139 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.Compactor(225): Compacting 4b1059e84c3c49acb94d51939ca6df50, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=181, earliestPutTs=1732378486971 2024-11-23T16:14:50,139 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.Compactor(225): Compacting 061f677f613e494fa09bff323886b641, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=181, earliestPutTs=1732378486971 2024-11-23T16:14:50,140 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.Compactor(225): Compacting 0027e9b63c7b41239c9f6219eac43145, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1732378488150 2024-11-23T16:14:50,140 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.Compactor(225): Compacting 46b14146244e4c5b85d34e3e83c18b9d, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1732378488150 2024-11-23T16:14:50,160 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5b34e731541fe76c476f23c5dc5aa28e#A#compaction#225 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-23T16:14:50,160 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/.tmp/A/dae109f61f604db98904191b015c8720 is 50, key is test_row_0/A:col10/1732378488880/Put/seqid=0 2024-11-23T16:14:50,161 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5b34e731541fe76c476f23c5dc5aa28e#B#compaction#226 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T16:14:50,162 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/.tmp/B/d708b428fb784ae9b2fd4aed7439b615 is 50, key is test_row_0/B:col10/1732378488880/Put/seqid=0 2024-11-23T16:14:50,178 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742095_1271 (size=12527) 2024-11-23T16:14:50,184 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/.tmp/A/dae109f61f604db98904191b015c8720 as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/A/dae109f61f604db98904191b015c8720 2024-11-23T16:14:50,186 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742096_1272 (size=12527) 2024-11-23T16:14:50,191 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 5b34e731541fe76c476f23c5dc5aa28e/A of 5b34e731541fe76c476f23c5dc5aa28e into dae109f61f604db98904191b015c8720(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T16:14:50,191 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5b34e731541fe76c476f23c5dc5aa28e: 2024-11-23T16:14:50,191 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., storeName=5b34e731541fe76c476f23c5dc5aa28e/A, priority=12, startTime=1732378490136; duration=0sec 2024-11-23T16:14:50,192 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T16:14:50,192 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5b34e731541fe76c476f23c5dc5aa28e:A 2024-11-23T16:14:50,192 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-23T16:14:50,194 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48794 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-23T16:14:50,194 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HStore(1541): 5b34e731541fe76c476f23c5dc5aa28e/C is initiating minor compaction (all files) 2024-11-23T16:14:50,194 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5b34e731541fe76c476f23c5dc5aa28e/C in TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e. 2024-11-23T16:14:50,194 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/C/367246ca68e04b1fb74befd857be030e, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/C/a6bb103255e34ed98332935e32943a73, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/C/5e04e969a5e84e1da2019df4463e5336, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/C/5de3d97f873c4881b6a1898aff51ec7a] into tmpdir=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/.tmp, totalSize=47.7 K 2024-11-23T16:14:50,194 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.Compactor(225): Compacting 367246ca68e04b1fb74befd857be030e, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1732378484082 2024-11-23T16:14:50,195 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.Compactor(225): Compacting a6bb103255e34ed98332935e32943a73, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1732378486258 2024-11-23T16:14:50,195 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5e04e969a5e84e1da2019df4463e5336, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=181, earliestPutTs=1732378486971 2024-11-23T16:14:50,195 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5de3d97f873c4881b6a1898aff51ec7a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1732378488150 2024-11-23T16:14:50,208 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5b34e731541fe76c476f23c5dc5aa28e#C#compaction#227 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T16:14:50,208 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/.tmp/C/53a0278513ee4759aa44605b9735eebb is 50, key is test_row_0/C:col10/1732378488880/Put/seqid=0 2024-11-23T16:14:50,224 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742097_1273 (size=12527) 2024-11-23T16:14:50,226 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-23T16:14:50,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e. 2024-11-23T16:14:50,226 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2902): Flushing 5b34e731541fe76c476f23c5dc5aa28e 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-23T16:14:50,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5b34e731541fe76c476f23c5dc5aa28e, store=A 2024-11-23T16:14:50,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-23T16:14:50,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5b34e731541fe76c476f23c5dc5aa28e, store=B 2024-11-23T16:14:50,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-23T16:14:50,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5b34e731541fe76c476f23c5dc5aa28e, store=C 2024-11-23T16:14:50,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-23T16:14:50,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/.tmp/A/adea4d701c414fb38fe3cb3cc61e0a4b is 50, key is test_row_0/A:col10/1732378488901/Put/seqid=0 2024-11-23T16:14:50,238 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742098_1274 (size=12151) 2024-11-23T16:14:50,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(8855): Flush requested on 5b34e731541fe76c476f23c5dc5aa28e 2024-11-23T16:14:50,311 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e. as already flushing 2024-11-23T16:14:50,339 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:50,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50480 deadline: 1732378550333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:50,339 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:50,339 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:50,340 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:14:50,345 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:50,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50424 deadline: 1732378550339, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:50,346 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:50,346 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:50,346 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:14:50,452 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:50,453 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:50,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50480 deadline: 1732378550448, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:50,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50424 deadline: 1732378550448, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:50,453 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:50,453 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:50,453 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:50,453 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:50,453 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:14:50,453 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:14:50,592 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/.tmp/B/d708b428fb784ae9b2fd4aed7439b615 as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/B/d708b428fb784ae9b2fd4aed7439b615 2024-11-23T16:14:50,597 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 5b34e731541fe76c476f23c5dc5aa28e/B of 5b34e731541fe76c476f23c5dc5aa28e into d708b428fb784ae9b2fd4aed7439b615(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T16:14:50,597 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5b34e731541fe76c476f23c5dc5aa28e: 2024-11-23T16:14:50,597 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., storeName=5b34e731541fe76c476f23c5dc5aa28e/B, priority=12, startTime=1732378490137; duration=0sec 2024-11-23T16:14:50,597 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T16:14:50,597 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5b34e731541fe76c476f23c5dc5aa28e:B 2024-11-23T16:14:50,629 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/.tmp/C/53a0278513ee4759aa44605b9735eebb as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/C/53a0278513ee4759aa44605b9735eebb 2024-11-23T16:14:50,635 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 5b34e731541fe76c476f23c5dc5aa28e/C of 5b34e731541fe76c476f23c5dc5aa28e into 53a0278513ee4759aa44605b9735eebb(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T16:14:50,635 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5b34e731541fe76c476f23c5dc5aa28e: 2024-11-23T16:14:50,635 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., storeName=5b34e731541fe76c476f23c5dc5aa28e/C, priority=12, startTime=1732378490137; duration=0sec 2024-11-23T16:14:50,635 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T16:14:50,635 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5b34e731541fe76c476f23c5dc5aa28e:C 2024-11-23T16:14:50,639 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=214 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/.tmp/A/adea4d701c414fb38fe3cb3cc61e0a4b 2024-11-23T16:14:50,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/.tmp/B/7677c1f3a1e34a458213645d1662ec7b is 50, key is test_row_0/B:col10/1732378488901/Put/seqid=0 2024-11-23T16:14:50,654 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742099_1275 (size=12151) 2024-11-23T16:14:50,661 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:50,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50480 deadline: 1732378550658, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:50,661 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:50,662 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:50,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50424 deadline: 1732378550658, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:50,662 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:50,662 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:14:50,662 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:50,662 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:50,662 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:14:50,968 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:50,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50480 deadline: 1732378550967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:50,969 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:50,969 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:50,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50424 deadline: 1732378550968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:50,969 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:50,969 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:14:50,969 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:50,969 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:50,969 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:14:51,056 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=214 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/.tmp/B/7677c1f3a1e34a458213645d1662ec7b 2024-11-23T16:14:51,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/.tmp/C/708501389e9047549b8fc4029c12ce34 is 50, key is test_row_0/C:col10/1732378488901/Put/seqid=0 2024-11-23T16:14:51,067 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742100_1276 (size=12151) 2024-11-23T16:14:51,090 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:51,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50454 deadline: 1732378551088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:51,091 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:51,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50462 deadline: 1732378551088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:51,091 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:51,091 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 1a7c5470b472,44715,1732378420375 for 'test_row_1' in 5b34e731541fe76c476f23c5dc5aa28e of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2163 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-23T16:14:51,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50486 deadline: 1732378551088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:51,092 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:51,092 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:51,092 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:14:51,091 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 1a7c5470b472,44715,1732378420375 for 'test_row_1' in 5b34e731541fe76c476f23c5dc5aa28e of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2162 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-23T16:14:51,092 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:51,092 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:51,092 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 1a7c5470b472,44715,1732378420375 for 'test_row_0' in 5b34e731541fe76c476f23c5dc5aa28e of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2161 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-23T16:14:51,092 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:14:51,092 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:51,092 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:51,092 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:14:51,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-23T16:14:51,468 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=214 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/.tmp/C/708501389e9047549b8fc4029c12ce34 2024-11-23T16:14:51,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/.tmp/A/adea4d701c414fb38fe3cb3cc61e0a4b as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/A/adea4d701c414fb38fe3cb3cc61e0a4b 2024-11-23T16:14:51,477 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/A/adea4d701c414fb38fe3cb3cc61e0a4b, entries=150, sequenceid=214, filesize=11.9 K 2024-11-23T16:14:51,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/.tmp/B/7677c1f3a1e34a458213645d1662ec7b as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/B/7677c1f3a1e34a458213645d1662ec7b 2024-11-23T16:14:51,481 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/B/7677c1f3a1e34a458213645d1662ec7b, entries=150, sequenceid=214, filesize=11.9 K 2024-11-23T16:14:51,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/.tmp/C/708501389e9047549b8fc4029c12ce34 as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/C/708501389e9047549b8fc4029c12ce34 2024-11-23T16:14:51,482 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:51,482 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:51,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50480 deadline: 1732378551478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:51,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50424 deadline: 1732378551478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:51,483 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:51,483 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:51,483 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:51,483 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:51,483 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:14:51,483 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:14:51,486 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/C/708501389e9047549b8fc4029c12ce34, entries=150, sequenceid=214, filesize=11.9 K 2024-11-23T16:14:51,487 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 5b34e731541fe76c476f23c5dc5aa28e in 1261ms, sequenceid=214, compaction requested=false 2024-11-23T16:14:51,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2603): Flush status journal for 5b34e731541fe76c476f23c5dc5aa28e: 2024-11-23T16:14:51,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e. 2024-11-23T16:14:51,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=69 2024-11-23T16:14:51,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.HMaster(4169): Remote procedure done, pid=69 2024-11-23T16:14:51,490 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=69, resume processing ppid=68 2024-11-23T16:14:51,490 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=69, ppid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.4870 sec 2024-11-23T16:14:51,492 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees in 2.4920 sec 2024-11-23T16:14:52,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(8855): Flush requested on 5b34e731541fe76c476f23c5dc5aa28e 2024-11-23T16:14:52,500 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5b34e731541fe76c476f23c5dc5aa28e 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-23T16:14:52,500 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5b34e731541fe76c476f23c5dc5aa28e, store=A 2024-11-23T16:14:52,500 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-23T16:14:52,500 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5b34e731541fe76c476f23c5dc5aa28e, store=B 2024-11-23T16:14:52,500 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-23T16:14:52,500 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5b34e731541fe76c476f23c5dc5aa28e, store=C 2024-11-23T16:14:52,500 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-23T16:14:52,505 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/.tmp/A/e0c94f0e49ff4b56890f7c49bab74512 is 50, key is test_row_0/A:col10/1732378490331/Put/seqid=0 2024-11-23T16:14:52,509 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742101_1277 (size=14541) 2024-11-23T16:14:52,510 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=229 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/.tmp/A/e0c94f0e49ff4b56890f7c49bab74512 2024-11-23T16:14:52,518 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/.tmp/B/77da15054a8849ee8605755feea9cc58 is 50, key is test_row_0/B:col10/1732378490331/Put/seqid=0 2024-11-23T16:14:52,522 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742102_1278 (size=12151) 2024-11-23T16:14:52,544 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:52,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50424 deadline: 1732378552539, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:52,544 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:52,544 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:52,544 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:14:52,548 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:52,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50480 deadline: 1732378552543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:52,548 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:52,548 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:52,548 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:14:52,651 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:52,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50424 deadline: 1732378552647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:52,652 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:52,652 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:52,652 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:14:52,661 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:52,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50480 deadline: 1732378552658, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:52,661 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:52,661 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:52,661 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:14:52,861 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:52,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50424 deadline: 1732378552858, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:52,862 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:52,862 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:52,862 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:14:52,871 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:52,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50480 deadline: 1732378552868, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:52,871 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:52,871 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:52,871 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:14:52,922 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=229 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/.tmp/B/77da15054a8849ee8605755feea9cc58 2024-11-23T16:14:52,931 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/.tmp/C/2cbcf9a84c3d40f78e6f8c2ccce6ef4f is 50, key is test_row_0/C:col10/1732378490331/Put/seqid=0 2024-11-23T16:14:52,935 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742103_1279 (size=12151) 2024-11-23T16:14:53,101 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:53,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50454 deadline: 1732378553098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:53,101 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 1a7c5470b472,44715,1732378420375 for 'test_row_1' in 5b34e731541fe76c476f23c5dc5aa28e of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4173 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-23T16:14:53,102 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:53,102 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:53,102 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:14:53,113 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:53,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50486 deadline: 1732378553108, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:53,113 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 1a7c5470b472,44715,1732378420375 for 'test_row_0' in 5b34e731541fe76c476f23c5dc5aa28e of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4182 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-23T16:14:53,114 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:53,114 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:53,114 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:14:53,119 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:53,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50462 deadline: 1732378553118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:53,120 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 1a7c5470b472,44715,1732378420375 for 'test_row_1' in 5b34e731541fe76c476f23c5dc5aa28e of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4190 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-23T16:14:53,121 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:53,121 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:53,121 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:14:53,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-23T16:14:53,158 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-23T16:14:53,159 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-23T16:14:53,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] procedure2.ProcedureExecutor(1139): Stored pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees 2024-11-23T16:14:53,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-23T16:14:53,162 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-23T16:14:53,163 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-23T16:14:53,163 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=71, ppid=70, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-23T16:14:53,171 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:53,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50424 deadline: 1732378553167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:53,172 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:53,172 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:53,172 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:14:53,181 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:53,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50480 deadline: 1732378553178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:53,182 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:53,182 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:53,182 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:14:53,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-23T16:14:53,316 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-23T16:14:53,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e. 2024-11-23T16:14:53,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e. as already flushing 2024-11-23T16:14:53,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e. 2024-11-23T16:14:53,316 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 5b34e731541fe76c476f23c5dc5aa28e, NAME => 'TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:14:53,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 5b34e731541fe76c476f23c5dc5aa28e, NAME => 'TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:14:53,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5b34e731541fe76c476f23c5dc5aa28e, NAME => 'TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5b34e731541fe76c476f23c5dc5aa28e, NAME => 'TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:14:53,336 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=229 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/.tmp/C/2cbcf9a84c3d40f78e6f8c2ccce6ef4f 2024-11-23T16:14:53,341 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/.tmp/A/e0c94f0e49ff4b56890f7c49bab74512 as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/A/e0c94f0e49ff4b56890f7c49bab74512 2024-11-23T16:14:53,346 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/A/e0c94f0e49ff4b56890f7c49bab74512, entries=200, sequenceid=229, filesize=14.2 K 2024-11-23T16:14:53,346 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/.tmp/B/77da15054a8849ee8605755feea9cc58 as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/B/77da15054a8849ee8605755feea9cc58 2024-11-23T16:14:53,350 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/B/77da15054a8849ee8605755feea9cc58, entries=150, sequenceid=229, filesize=11.9 K 2024-11-23T16:14:53,352 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/.tmp/C/2cbcf9a84c3d40f78e6f8c2ccce6ef4f as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/C/2cbcf9a84c3d40f78e6f8c2ccce6ef4f 2024-11-23T16:14:53,357 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/C/2cbcf9a84c3d40f78e6f8c2ccce6ef4f, entries=150, sequenceid=229, filesize=11.9 K 2024-11-23T16:14:53,358 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 5b34e731541fe76c476f23c5dc5aa28e in 858ms, sequenceid=229, compaction requested=true 2024-11-23T16:14:53,358 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5b34e731541fe76c476f23c5dc5aa28e: 2024-11-23T16:14:53,358 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T16:14:53,359 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5b34e731541fe76c476f23c5dc5aa28e:A, priority=-2147483648, current under compaction store size is 1 2024-11-23T16:14:53,359 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T16:14:53,359 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5b34e731541fe76c476f23c5dc5aa28e:B, priority=-2147483648, current under compaction store size is 2 2024-11-23T16:14:53,359 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T16:14:53,359 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5b34e731541fe76c476f23c5dc5aa28e:C, priority=-2147483648, current under compaction store size is 3 2024-11-23T16:14:53,359 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-23T16:14:53,359 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T16:14:53,360 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36829 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T16:14:53,360 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HStore(1541): 5b34e731541fe76c476f23c5dc5aa28e/B is initiating minor compaction (all files) 2024-11-23T16:14:53,360 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5b34e731541fe76c476f23c5dc5aa28e/B in TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e. 2024-11-23T16:14:53,361 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/B/d708b428fb784ae9b2fd4aed7439b615, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/B/7677c1f3a1e34a458213645d1662ec7b, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/B/77da15054a8849ee8605755feea9cc58] into tmpdir=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/.tmp, totalSize=36.0 K 2024-11-23T16:14:53,361 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39219 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T16:14:53,361 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HStore(1541): 5b34e731541fe76c476f23c5dc5aa28e/A is initiating minor compaction (all files) 2024-11-23T16:14:53,361 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5b34e731541fe76c476f23c5dc5aa28e/A in TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e. 2024-11-23T16:14:53,361 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/A/dae109f61f604db98904191b015c8720, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/A/adea4d701c414fb38fe3cb3cc61e0a4b, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/A/e0c94f0e49ff4b56890f7c49bab74512] into tmpdir=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/.tmp, totalSize=38.3 K 2024-11-23T16:14:53,362 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.Compactor(225): Compacting d708b428fb784ae9b2fd4aed7439b615, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1732378488150 2024-11-23T16:14:53,362 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.Compactor(225): Compacting dae109f61f604db98904191b015c8720, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1732378488150 2024-11-23T16:14:53,362 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.Compactor(225): Compacting 7677c1f3a1e34a458213645d1662ec7b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1732378488901 2024-11-23T16:14:53,362 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.Compactor(225): Compacting adea4d701c414fb38fe3cb3cc61e0a4b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1732378488901 2024-11-23T16:14:53,362 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.Compactor(225): Compacting 77da15054a8849ee8605755feea9cc58, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=229, earliestPutTs=1732378490331 2024-11-23T16:14:53,363 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.Compactor(225): Compacting e0c94f0e49ff4b56890f7c49bab74512, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=229, earliestPutTs=1732378490324 2024-11-23T16:14:53,371 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5b34e731541fe76c476f23c5dc5aa28e#A#compaction#234 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-23T16:14:53,372 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5b34e731541fe76c476f23c5dc5aa28e#B#compaction#235 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T16:14:53,373 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/.tmp/A/eb4498eb5bb84238ae869bcfaf52f8d9 is 50, key is test_row_0/A:col10/1732378490331/Put/seqid=0 2024-11-23T16:14:53,373 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/.tmp/B/329fd3cf8b0f4223a7665f8704a3ea99 is 50, key is test_row_0/B:col10/1732378490331/Put/seqid=0 2024-11-23T16:14:53,382 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742104_1280 (size=12629) 2024-11-23T16:14:53,387 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/.tmp/A/eb4498eb5bb84238ae869bcfaf52f8d9 as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/A/eb4498eb5bb84238ae869bcfaf52f8d9 2024-11-23T16:14:53,393 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5b34e731541fe76c476f23c5dc5aa28e/A of 5b34e731541fe76c476f23c5dc5aa28e into eb4498eb5bb84238ae869bcfaf52f8d9(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T16:14:53,393 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5b34e731541fe76c476f23c5dc5aa28e: 2024-11-23T16:14:53,393 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., storeName=5b34e731541fe76c476f23c5dc5aa28e/A, priority=13, startTime=1732378493358; duration=0sec 2024-11-23T16:14:53,393 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T16:14:53,393 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5b34e731541fe76c476f23c5dc5aa28e:A 2024-11-23T16:14:53,393 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T16:14:53,394 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36829 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T16:14:53,394 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HStore(1541): 5b34e731541fe76c476f23c5dc5aa28e/C is initiating minor compaction (all files) 2024-11-23T16:14:53,396 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5b34e731541fe76c476f23c5dc5aa28e/C in TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e. 2024-11-23T16:14:53,396 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/C/53a0278513ee4759aa44605b9735eebb, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/C/708501389e9047549b8fc4029c12ce34, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/C/2cbcf9a84c3d40f78e6f8c2ccce6ef4f] into tmpdir=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/.tmp, totalSize=36.0 K 2024-11-23T16:14:53,397 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.Compactor(225): Compacting 53a0278513ee4759aa44605b9735eebb, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1732378488150 2024-11-23T16:14:53,397 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.Compactor(225): Compacting 708501389e9047549b8fc4029c12ce34, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1732378488901 2024-11-23T16:14:53,397 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2cbcf9a84c3d40f78e6f8c2ccce6ef4f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=229, earliestPutTs=1732378490331 2024-11-23T16:14:53,401 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742105_1281 (size=12629) 2024-11-23T16:14:53,409 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/.tmp/B/329fd3cf8b0f4223a7665f8704a3ea99 as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/B/329fd3cf8b0f4223a7665f8704a3ea99 2024-11-23T16:14:53,414 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5b34e731541fe76c476f23c5dc5aa28e/B of 5b34e731541fe76c476f23c5dc5aa28e into 329fd3cf8b0f4223a7665f8704a3ea99(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T16:14:53,414 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5b34e731541fe76c476f23c5dc5aa28e: 2024-11-23T16:14:53,414 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., storeName=5b34e731541fe76c476f23c5dc5aa28e/B, priority=13, startTime=1732378493359; duration=0sec 2024-11-23T16:14:53,414 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T16:14:53,414 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5b34e731541fe76c476f23c5dc5aa28e:B 2024-11-23T16:14:53,419 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5b34e731541fe76c476f23c5dc5aa28e#C#compaction#236 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T16:14:53,419 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/.tmp/C/89c4a913ef464966be462584a775156d is 50, key is test_row_0/C:col10/1732378490331/Put/seqid=0 2024-11-23T16:14:53,427 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742106_1282 (size=12629) 2024-11-23T16:14:53,433 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/.tmp/C/89c4a913ef464966be462584a775156d as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/C/89c4a913ef464966be462584a775156d 2024-11-23T16:14:53,438 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5b34e731541fe76c476f23c5dc5aa28e/C of 5b34e731541fe76c476f23c5dc5aa28e into 89c4a913ef464966be462584a775156d(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T16:14:53,438 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5b34e731541fe76c476f23c5dc5aa28e: 2024-11-23T16:14:53,438 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., storeName=5b34e731541fe76c476f23c5dc5aa28e/C, priority=13, startTime=1732378493359; duration=0sec 2024-11-23T16:14:53,438 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T16:14:53,438 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5b34e731541fe76c476f23c5dc5aa28e:C 2024-11-23T16:14:53,469 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-23T16:14:53,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e. 2024-11-23T16:14:53,469 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2902): Flushing 5b34e731541fe76c476f23c5dc5aa28e 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-23T16:14:53,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5b34e731541fe76c476f23c5dc5aa28e, store=A 2024-11-23T16:14:53,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-23T16:14:53,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5b34e731541fe76c476f23c5dc5aa28e, store=B 2024-11-23T16:14:53,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-23T16:14:53,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5b34e731541fe76c476f23c5dc5aa28e, store=C 2024-11-23T16:14:53,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-23T16:14:53,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/.tmp/A/7a25f247685b4423afcc6d516b6f0c46 is 50, key is test_row_0/A:col10/1732378492538/Put/seqid=0 2024-11-23T16:14:53,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-23T16:14:53,478 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742107_1283 (size=12151) 2024-11-23T16:14:53,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(8855): Flush requested on 5b34e731541fe76c476f23c5dc5aa28e 2024-11-23T16:14:53,680 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e. as already flushing 2024-11-23T16:14:53,704 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:53,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50480 deadline: 1732378553701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:53,705 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:53,705 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:53,705 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:53,705 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:14:53,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50424 deadline: 1732378553702, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:53,705 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:53,705 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:53,705 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:14:53,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-23T16:14:53,811 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:53,811 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:53,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50480 deadline: 1732378553808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:53,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50424 deadline: 1732378553808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:53,812 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:53,812 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:53,812 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:53,812 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:14:53,812 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:53,812 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:14:53,879 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=251 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/.tmp/A/7a25f247685b4423afcc6d516b6f0c46 2024-11-23T16:14:53,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/.tmp/B/31dba82a738d4286be490e41d057aefc is 50, key is test_row_0/B:col10/1732378492538/Put/seqid=0 2024-11-23T16:14:53,891 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742108_1284 (size=12151) 2024-11-23T16:14:54,020 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:54,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50424 deadline: 1732378554018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:54,020 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:54,020 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:54,020 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:54,020 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:14:54,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50480 deadline: 1732378554018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:54,021 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:54,021 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:54,021 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:14:54,292 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=251 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/.tmp/B/31dba82a738d4286be490e41d057aefc 2024-11-23T16:14:54,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-23T16:14:54,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/.tmp/C/21f8ac0b64214560bb2e0b208542f540 is 50, key is test_row_0/C:col10/1732378492538/Put/seqid=0 2024-11-23T16:14:54,304 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742109_1285 (size=12151) 2024-11-23T16:14:54,330 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:54,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50424 deadline: 1732378554327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:54,330 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:54,330 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:54,330 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:54,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50480 deadline: 1732378554327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:54,330 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:14:54,331 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:54,331 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:54,331 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:14:54,705 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=251 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/.tmp/C/21f8ac0b64214560bb2e0b208542f540 2024-11-23T16:14:54,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/.tmp/A/7a25f247685b4423afcc6d516b6f0c46 as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/A/7a25f247685b4423afcc6d516b6f0c46 2024-11-23T16:14:54,714 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/A/7a25f247685b4423afcc6d516b6f0c46, entries=150, sequenceid=251, filesize=11.9 K 2024-11-23T16:14:54,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/.tmp/B/31dba82a738d4286be490e41d057aefc as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/B/31dba82a738d4286be490e41d057aefc 2024-11-23T16:14:54,718 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/B/31dba82a738d4286be490e41d057aefc, entries=150, sequenceid=251, filesize=11.9 K 2024-11-23T16:14:54,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/.tmp/C/21f8ac0b64214560bb2e0b208542f540 as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/C/21f8ac0b64214560bb2e0b208542f540 2024-11-23T16:14:54,726 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/C/21f8ac0b64214560bb2e0b208542f540, entries=150, sequenceid=251, filesize=11.9 K 2024-11-23T16:14:54,727 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 5b34e731541fe76c476f23c5dc5aa28e in 1258ms, sequenceid=251, compaction requested=false 2024-11-23T16:14:54,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2603): Flush status journal for 5b34e731541fe76c476f23c5dc5aa28e: 2024-11-23T16:14:54,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e. 2024-11-23T16:14:54,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=71 2024-11-23T16:14:54,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.HMaster(4169): Remote procedure done, pid=71 2024-11-23T16:14:54,731 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=71, resume processing ppid=70 2024-11-23T16:14:54,731 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=71, ppid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5660 sec 2024-11-23T16:14:54,733 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees in 1.5720 sec 2024-11-23T16:14:54,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(8855): Flush requested on 5b34e731541fe76c476f23c5dc5aa28e 2024-11-23T16:14:54,840 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5b34e731541fe76c476f23c5dc5aa28e 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-23T16:14:54,840 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5b34e731541fe76c476f23c5dc5aa28e, store=A 2024-11-23T16:14:54,840 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-23T16:14:54,841 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5b34e731541fe76c476f23c5dc5aa28e, store=B 2024-11-23T16:14:54,841 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-23T16:14:54,841 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5b34e731541fe76c476f23c5dc5aa28e, store=C 2024-11-23T16:14:54,841 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-23T16:14:54,845 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/.tmp/A/b15e6612f7c347b98df70d673799702e is 50, key is test_row_0/A:col10/1732378494839/Put/seqid=0 2024-11-23T16:14:54,850 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742110_1286 (size=12301) 2024-11-23T16:14:54,879 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:54,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50480 deadline: 1732378554874, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:54,879 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:54,879 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:54,879 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:14:54,881 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:54,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50424 deadline: 1732378554877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:54,881 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:54,881 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:54,881 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:14:54,991 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:54,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50480 deadline: 1732378554987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:54,991 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:54,991 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:54,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50424 deadline: 1732378554987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:54,991 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:54,991 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:14:54,992 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:54,992 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:54,992 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:14:55,201 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:55,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50480 deadline: 1732378555198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:55,201 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:55,201 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:55,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50424 deadline: 1732378555198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:55,201 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:55,201 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:14:55,202 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:55,202 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:55,202 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:14:55,250 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=266 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/.tmp/A/b15e6612f7c347b98df70d673799702e 2024-11-23T16:14:55,258 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/.tmp/B/3d4ef229f7684d3bb4820557740d2398 is 50, key is test_row_0/B:col10/1732378494839/Put/seqid=0 2024-11-23T16:14:55,261 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742111_1287 (size=12301) 2024-11-23T16:14:55,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-23T16:14:55,319 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-23T16:14:55,319 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-23T16:14:55,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] procedure2.ProcedureExecutor(1139): Stored pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees 2024-11-23T16:14:55,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-23T16:14:55,321 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-23T16:14:55,322 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-23T16:14:55,322 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=73, ppid=72, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-23T16:14:55,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-23T16:14:55,474 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-23T16:14:55,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e. 2024-11-23T16:14:55,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e. as already flushing 2024-11-23T16:14:55,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e. 2024-11-23T16:14:55,475 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 5b34e731541fe76c476f23c5dc5aa28e, NAME => 'TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:14:55,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 5b34e731541fe76c476f23c5dc5aa28e, NAME => 'TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:14:55,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5b34e731541fe76c476f23c5dc5aa28e, NAME => 'TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5b34e731541fe76c476f23c5dc5aa28e, NAME => 'TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:14:55,511 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:55,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50480 deadline: 1732378555508, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:55,512 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:55,512 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:55,512 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:55,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50424 deadline: 1732378555508, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:55,512 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:14:55,512 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:55,512 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:55,512 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:14:55,627 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-23T16:14:55,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e. 2024-11-23T16:14:55,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e. as already flushing 2024-11-23T16:14:55,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e. 2024-11-23T16:14:55,628 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 5b34e731541fe76c476f23c5dc5aa28e, NAME => 'TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:14:55,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 5b34e731541fe76c476f23c5dc5aa28e, NAME => 'TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:14:55,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5b34e731541fe76c476f23c5dc5aa28e, NAME => 'TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5b34e731541fe76c476f23c5dc5aa28e, NAME => 'TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:14:55,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-23T16:14:55,662 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=266 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/.tmp/B/3d4ef229f7684d3bb4820557740d2398 2024-11-23T16:14:55,670 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/.tmp/C/f4e3a661402e40ea8860b469da3b9331 is 50, key is test_row_0/C:col10/1732378494839/Put/seqid=0 2024-11-23T16:14:55,673 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742112_1288 (size=12301) 2024-11-23T16:14:55,781 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-23T16:14:55,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e. 2024-11-23T16:14:55,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e. as already flushing 2024-11-23T16:14:55,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e. 2024-11-23T16:14:55,781 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 5b34e731541fe76c476f23c5dc5aa28e, NAME => 'TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:14:55,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 5b34e731541fe76c476f23c5dc5aa28e, NAME => 'TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:14:55,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5b34e731541fe76c476f23c5dc5aa28e, NAME => 'TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5b34e731541fe76c476f23c5dc5aa28e, NAME => 'TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:14:55,934 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-23T16:14:55,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e. 2024-11-23T16:14:55,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e. as already flushing 2024-11-23T16:14:55,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e. 2024-11-23T16:14:55,934 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 5b34e731541fe76c476f23c5dc5aa28e, NAME => 'TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:14:55,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 5b34e731541fe76c476f23c5dc5aa28e, NAME => 'TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:14:55,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5b34e731541fe76c476f23c5dc5aa28e, NAME => 'TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5b34e731541fe76c476f23c5dc5aa28e, NAME => 'TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:14:55,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-23T16:14:56,021 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:56,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50480 deadline: 1732378556018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:56,021 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:56,021 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:56,021 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:56,021 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:14:56,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50424 deadline: 1732378556018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:56,022 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:56,022 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:56,022 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:14:56,074 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=266 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/.tmp/C/f4e3a661402e40ea8860b469da3b9331 2024-11-23T16:14:56,079 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/.tmp/A/b15e6612f7c347b98df70d673799702e as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/A/b15e6612f7c347b98df70d673799702e 2024-11-23T16:14:56,083 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/A/b15e6612f7c347b98df70d673799702e, entries=150, sequenceid=266, filesize=12.0 K 2024-11-23T16:14:56,084 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/.tmp/B/3d4ef229f7684d3bb4820557740d2398 as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/B/3d4ef229f7684d3bb4820557740d2398 2024-11-23T16:14:56,087 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-23T16:14:56,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e. 2024-11-23T16:14:56,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e. as already flushing 2024-11-23T16:14:56,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e. 2024-11-23T16:14:56,087 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 5b34e731541fe76c476f23c5dc5aa28e, NAME => 'TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:14:56,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 5b34e731541fe76c476f23c5dc5aa28e, NAME => 'TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:14:56,088 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/B/3d4ef229f7684d3bb4820557740d2398, entries=150, sequenceid=266, filesize=12.0 K 2024-11-23T16:14:56,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5b34e731541fe76c476f23c5dc5aa28e, NAME => 'TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5b34e731541fe76c476f23c5dc5aa28e, NAME => 'TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:14:56,089 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/.tmp/C/f4e3a661402e40ea8860b469da3b9331 as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/C/f4e3a661402e40ea8860b469da3b9331 2024-11-23T16:14:56,093 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/C/f4e3a661402e40ea8860b469da3b9331, entries=150, sequenceid=266, filesize=12.0 K 2024-11-23T16:14:56,094 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 5b34e731541fe76c476f23c5dc5aa28e in 1254ms, sequenceid=266, compaction requested=true 2024-11-23T16:14:56,094 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5b34e731541fe76c476f23c5dc5aa28e: 2024-11-23T16:14:56,094 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5b34e731541fe76c476f23c5dc5aa28e:A, priority=-2147483648, current under compaction store size is 1 2024-11-23T16:14:56,094 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T16:14:56,094 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5b34e731541fe76c476f23c5dc5aa28e:B, priority=-2147483648, current under compaction store size is 2 2024-11-23T16:14:56,094 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T16:14:56,094 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T16:14:56,094 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5b34e731541fe76c476f23c5dc5aa28e:C, priority=-2147483648, current under compaction store size is 3 2024-11-23T16:14:56,094 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T16:14:56,094 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T16:14:56,095 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37081 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T16:14:56,095 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37081 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T16:14:56,095 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HStore(1541): 5b34e731541fe76c476f23c5dc5aa28e/B is initiating minor compaction (all files) 2024-11-23T16:14:56,095 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HStore(1541): 5b34e731541fe76c476f23c5dc5aa28e/A is initiating minor compaction (all files) 2024-11-23T16:14:56,095 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5b34e731541fe76c476f23c5dc5aa28e/B in TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e. 2024-11-23T16:14:56,095 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5b34e731541fe76c476f23c5dc5aa28e/A in TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e. 2024-11-23T16:14:56,095 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/B/329fd3cf8b0f4223a7665f8704a3ea99, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/B/31dba82a738d4286be490e41d057aefc, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/B/3d4ef229f7684d3bb4820557740d2398] into tmpdir=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/.tmp, totalSize=36.2 K 2024-11-23T16:14:56,095 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/A/eb4498eb5bb84238ae869bcfaf52f8d9, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/A/7a25f247685b4423afcc6d516b6f0c46, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/A/b15e6612f7c347b98df70d673799702e] into tmpdir=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/.tmp, totalSize=36.2 K 2024-11-23T16:14:56,096 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.Compactor(225): Compacting 329fd3cf8b0f4223a7665f8704a3ea99, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=229, earliestPutTs=1732378490331 2024-11-23T16:14:56,096 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.Compactor(225): Compacting eb4498eb5bb84238ae869bcfaf52f8d9, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=229, earliestPutTs=1732378490331 2024-11-23T16:14:56,096 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.Compactor(225): Compacting 31dba82a738d4286be490e41d057aefc, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=251, earliestPutTs=1732378492526 2024-11-23T16:14:56,096 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7a25f247685b4423afcc6d516b6f0c46, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=251, earliestPutTs=1732378492526 2024-11-23T16:14:56,097 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.Compactor(225): Compacting 3d4ef229f7684d3bb4820557740d2398, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=266, earliestPutTs=1732378493696 2024-11-23T16:14:56,097 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.Compactor(225): Compacting b15e6612f7c347b98df70d673799702e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=266, earliestPutTs=1732378493696 2024-11-23T16:14:56,106 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5b34e731541fe76c476f23c5dc5aa28e#A#compaction#243 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T16:14:56,106 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/.tmp/A/40635fabeefe44d7a3b73195943c2d4b is 50, key is test_row_0/A:col10/1732378494839/Put/seqid=0 2024-11-23T16:14:56,119 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5b34e731541fe76c476f23c5dc5aa28e#B#compaction#244 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T16:14:56,119 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742113_1289 (size=12881) 2024-11-23T16:14:56,119 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/.tmp/B/26a27afba72d4f37b9aeed9a4a861c0a is 50, key is test_row_0/B:col10/1732378494839/Put/seqid=0 2024-11-23T16:14:56,125 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742114_1290 (size=12881) 2024-11-23T16:14:56,141 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/.tmp/A/40635fabeefe44d7a3b73195943c2d4b as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/A/40635fabeefe44d7a3b73195943c2d4b 2024-11-23T16:14:56,149 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5b34e731541fe76c476f23c5dc5aa28e/A of 5b34e731541fe76c476f23c5dc5aa28e into 40635fabeefe44d7a3b73195943c2d4b(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T16:14:56,149 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5b34e731541fe76c476f23c5dc5aa28e: 2024-11-23T16:14:56,149 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., storeName=5b34e731541fe76c476f23c5dc5aa28e/A, priority=13, startTime=1732378496094; duration=0sec 2024-11-23T16:14:56,149 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T16:14:56,149 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5b34e731541fe76c476f23c5dc5aa28e:A 2024-11-23T16:14:56,149 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T16:14:56,153 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37081 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T16:14:56,153 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HStore(1541): 5b34e731541fe76c476f23c5dc5aa28e/C is initiating minor compaction (all files) 2024-11-23T16:14:56,153 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5b34e731541fe76c476f23c5dc5aa28e/C in TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e. 2024-11-23T16:14:56,153 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/C/89c4a913ef464966be462584a775156d, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/C/21f8ac0b64214560bb2e0b208542f540, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/C/f4e3a661402e40ea8860b469da3b9331] into tmpdir=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/.tmp, totalSize=36.2 K 2024-11-23T16:14:56,154 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.Compactor(225): Compacting 89c4a913ef464966be462584a775156d, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=229, earliestPutTs=1732378490331 2024-11-23T16:14:56,154 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.Compactor(225): Compacting 21f8ac0b64214560bb2e0b208542f540, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=251, earliestPutTs=1732378492526 2024-11-23T16:14:56,155 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.Compactor(225): Compacting f4e3a661402e40ea8860b469da3b9331, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=266, earliestPutTs=1732378493696 2024-11-23T16:14:56,163 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5b34e731541fe76c476f23c5dc5aa28e#C#compaction#245 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T16:14:56,164 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/.tmp/C/134baf646ea8406ca000d85dc5aa31bf is 50, key is test_row_0/C:col10/1732378494839/Put/seqid=0 2024-11-23T16:14:56,175 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742115_1291 (size=12881) 2024-11-23T16:14:56,182 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/.tmp/C/134baf646ea8406ca000d85dc5aa31bf as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/C/134baf646ea8406ca000d85dc5aa31bf 2024-11-23T16:14:56,188 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5b34e731541fe76c476f23c5dc5aa28e/C of 5b34e731541fe76c476f23c5dc5aa28e into 134baf646ea8406ca000d85dc5aa31bf(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T16:14:56,188 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5b34e731541fe76c476f23c5dc5aa28e: 2024-11-23T16:14:56,188 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., storeName=5b34e731541fe76c476f23c5dc5aa28e/C, priority=13, startTime=1732378496094; duration=0sec 2024-11-23T16:14:56,189 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T16:14:56,189 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5b34e731541fe76c476f23c5dc5aa28e:C 2024-11-23T16:14:56,240 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-23T16:14:56,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e. 2024-11-23T16:14:56,240 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2902): Flushing 5b34e731541fe76c476f23c5dc5aa28e 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-23T16:14:56,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5b34e731541fe76c476f23c5dc5aa28e, store=A 2024-11-23T16:14:56,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-23T16:14:56,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5b34e731541fe76c476f23c5dc5aa28e, store=B 2024-11-23T16:14:56,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-23T16:14:56,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5b34e731541fe76c476f23c5dc5aa28e, store=C 2024-11-23T16:14:56,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-23T16:14:56,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/.tmp/A/8bb3031477224a5eb9dd9cf281e38bc1 is 50, key is test_row_0/A:col10/1732378494873/Put/seqid=0 2024-11-23T16:14:56,251 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742116_1292 (size=12301) 2024-11-23T16:14:56,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-23T16:14:56,531 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/.tmp/B/26a27afba72d4f37b9aeed9a4a861c0a as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/B/26a27afba72d4f37b9aeed9a4a861c0a 2024-11-23T16:14:56,536 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5b34e731541fe76c476f23c5dc5aa28e/B of 5b34e731541fe76c476f23c5dc5aa28e into 26a27afba72d4f37b9aeed9a4a861c0a(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T16:14:56,536 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5b34e731541fe76c476f23c5dc5aa28e: 2024-11-23T16:14:56,536 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., storeName=5b34e731541fe76c476f23c5dc5aa28e/B, priority=13, startTime=1732378496094; duration=0sec 2024-11-23T16:14:56,536 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T16:14:56,536 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5b34e731541fe76c476f23c5dc5aa28e:B 2024-11-23T16:14:56,651 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=285 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/.tmp/A/8bb3031477224a5eb9dd9cf281e38bc1 2024-11-23T16:14:56,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/.tmp/B/e84a9105ca6d42c89fdd7381444e28b0 is 50, key is test_row_0/B:col10/1732378494873/Put/seqid=0 2024-11-23T16:14:56,664 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742117_1293 (size=12301) 2024-11-23T16:14:57,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(8855): Flush requested on 5b34e731541fe76c476f23c5dc5aa28e 2024-11-23T16:14:57,030 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e. as already flushing 2024-11-23T16:14:57,064 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=285 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/.tmp/B/e84a9105ca6d42c89fdd7381444e28b0 2024-11-23T16:14:57,066 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:57,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50424 deadline: 1732378557061, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:57,066 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:57,066 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:57,066 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:57,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50480 deadline: 1732378557064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:57,066 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:14:57,067 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:57,067 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:57,067 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:14:57,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/.tmp/C/ac1d22e427fe4f85bc8b10a2d854a907 is 50, key is test_row_0/C:col10/1732378494873/Put/seqid=0 2024-11-23T16:14:57,076 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742118_1294 (size=12301) 2024-11-23T16:14:57,131 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:57,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50454 deadline: 1732378557127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:57,131 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 1a7c5470b472,44715,1732378420375 for 'test_row_1' in 5b34e731541fe76c476f23c5dc5aa28e of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8203 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-23T16:14:57,132 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:57,132 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:57,132 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:14:57,139 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:57,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50462 deadline: 1732378557138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:57,140 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 1a7c5470b472,44715,1732378420375 for 'test_row_1' in 5b34e731541fe76c476f23c5dc5aa28e of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8210 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-23T16:14:57,140 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:57,140 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:57,141 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:14:57,151 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:57,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50486 deadline: 1732378557148, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:57,151 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 1a7c5470b472,44715,1732378420375 for 'test_row_0' in 5b34e731541fe76c476f23c5dc5aa28e of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8220 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-23T16:14:57,152 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:57,152 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:57,152 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:14:57,170 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:57,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50424 deadline: 1732378557167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:57,171 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:57,171 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:57,171 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:14:57,180 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:57,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50480 deadline: 1732378557178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:57,181 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:57,181 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:57,181 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:14:57,380 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:57,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50424 deadline: 1732378557378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:57,380 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:57,380 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:57,380 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:14:57,390 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:57,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50480 deadline: 1732378557388, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:57,390 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:57,390 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:57,390 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:14:57,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-23T16:14:57,476 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=285 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/.tmp/C/ac1d22e427fe4f85bc8b10a2d854a907 2024-11-23T16:14:57,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/.tmp/A/8bb3031477224a5eb9dd9cf281e38bc1 as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/A/8bb3031477224a5eb9dd9cf281e38bc1 2024-11-23T16:14:57,485 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/A/8bb3031477224a5eb9dd9cf281e38bc1, entries=150, sequenceid=285, filesize=12.0 K 2024-11-23T16:14:57,486 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/.tmp/B/e84a9105ca6d42c89fdd7381444e28b0 as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/B/e84a9105ca6d42c89fdd7381444e28b0 2024-11-23T16:14:57,490 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/B/e84a9105ca6d42c89fdd7381444e28b0, entries=150, sequenceid=285, filesize=12.0 K 2024-11-23T16:14:57,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/.tmp/C/ac1d22e427fe4f85bc8b10a2d854a907 as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/C/ac1d22e427fe4f85bc8b10a2d854a907 2024-11-23T16:14:57,495 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/C/ac1d22e427fe4f85bc8b10a2d854a907, entries=150, sequenceid=285, filesize=12.0 K 2024-11-23T16:14:57,496 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 5b34e731541fe76c476f23c5dc5aa28e in 1256ms, sequenceid=285, compaction requested=false 2024-11-23T16:14:57,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2603): Flush status journal for 5b34e731541fe76c476f23c5dc5aa28e: 2024-11-23T16:14:57,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e. 2024-11-23T16:14:57,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=73 2024-11-23T16:14:57,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.HMaster(4169): Remote procedure done, pid=73 2024-11-23T16:14:57,499 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=73, resume processing ppid=72 2024-11-23T16:14:57,499 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=73, ppid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1750 sec 2024-11-23T16:14:57,502 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees in 2.1810 sec 2024-11-23T16:14:57,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(8855): Flush requested on 5b34e731541fe76c476f23c5dc5aa28e 2024-11-23T16:14:57,691 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5b34e731541fe76c476f23c5dc5aa28e 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-23T16:14:57,691 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5b34e731541fe76c476f23c5dc5aa28e, store=A 2024-11-23T16:14:57,691 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-23T16:14:57,691 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5b34e731541fe76c476f23c5dc5aa28e, store=B 2024-11-23T16:14:57,691 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-23T16:14:57,691 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5b34e731541fe76c476f23c5dc5aa28e, store=C 2024-11-23T16:14:57,691 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-23T16:14:57,696 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/.tmp/A/f3cc850d44604796a355f0e2c5c5e0d7 is 50, key is test_row_0/A:col10/1732378497063/Put/seqid=0 2024-11-23T16:14:57,701 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742119_1295 (size=14741) 2024-11-23T16:14:57,728 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:57,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50424 deadline: 1732378557724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:57,729 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:57,729 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:57,729 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:14:57,729 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:57,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50480 deadline: 1732378557725, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:57,730 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:57,730 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:57,730 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:14:57,841 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:57,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50424 deadline: 1732378557838, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:57,842 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:57,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50480 deadline: 1732378557838, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:57,842 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:57,842 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:57,842 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:14:57,842 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:57,842 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:57,842 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:14:58,053 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:58,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50424 deadline: 1732378558048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:58,053 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:58,053 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:58,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50480 deadline: 1732378558049, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:58,053 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:58,053 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:14:58,054 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:58,054 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:58,054 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:14:58,102 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=302 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/.tmp/A/f3cc850d44604796a355f0e2c5c5e0d7 2024-11-23T16:14:58,112 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/.tmp/B/79dd470ab1854873b777ea2b1eb91d47 is 50, key is test_row_0/B:col10/1732378497063/Put/seqid=0 2024-11-23T16:14:58,118 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742120_1296 (size=12301) 2024-11-23T16:14:58,119 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=302 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/.tmp/B/79dd470ab1854873b777ea2b1eb91d47 2024-11-23T16:14:58,131 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/.tmp/C/5c18e55bdda740b793020357fca9c24a is 50, key is test_row_0/C:col10/1732378497063/Put/seqid=0 2024-11-23T16:14:58,139 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742121_1297 (size=12301) 2024-11-23T16:14:58,360 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:58,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50424 deadline: 1732378558358, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:58,361 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:58,361 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:58,361 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:58,361 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:14:58,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50480 deadline: 1732378558358, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:58,361 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:58,361 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:58,361 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:14:58,540 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=302 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/.tmp/C/5c18e55bdda740b793020357fca9c24a 2024-11-23T16:14:58,548 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/.tmp/A/f3cc850d44604796a355f0e2c5c5e0d7 as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/A/f3cc850d44604796a355f0e2c5c5e0d7 2024-11-23T16:14:58,553 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/A/f3cc850d44604796a355f0e2c5c5e0d7, entries=200, sequenceid=302, filesize=14.4 K 2024-11-23T16:14:58,554 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/.tmp/B/79dd470ab1854873b777ea2b1eb91d47 as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/B/79dd470ab1854873b777ea2b1eb91d47 2024-11-23T16:14:58,559 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/B/79dd470ab1854873b777ea2b1eb91d47, entries=150, sequenceid=302, filesize=12.0 K 2024-11-23T16:14:58,560 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/.tmp/C/5c18e55bdda740b793020357fca9c24a as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/C/5c18e55bdda740b793020357fca9c24a 2024-11-23T16:14:58,565 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/C/5c18e55bdda740b793020357fca9c24a, entries=150, sequenceid=302, filesize=12.0 K 2024-11-23T16:14:58,566 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 5b34e731541fe76c476f23c5dc5aa28e in 876ms, sequenceid=302, compaction requested=true 2024-11-23T16:14:58,566 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5b34e731541fe76c476f23c5dc5aa28e: 2024-11-23T16:14:58,567 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5b34e731541fe76c476f23c5dc5aa28e:A, priority=-2147483648, current under compaction store size is 1 2024-11-23T16:14:58,567 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T16:14:58,567 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5b34e731541fe76c476f23c5dc5aa28e:B, priority=-2147483648, current under compaction store size is 2 2024-11-23T16:14:58,567 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T16:14:58,567 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T16:14:58,567 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5b34e731541fe76c476f23c5dc5aa28e:C, priority=-2147483648, current under compaction store size is 3 2024-11-23T16:14:58,567 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T16:14:58,567 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T16:14:58,568 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39923 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T16:14:58,568 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HStore(1541): 5b34e731541fe76c476f23c5dc5aa28e/A is initiating minor compaction (all files) 2024-11-23T16:14:58,568 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5b34e731541fe76c476f23c5dc5aa28e/A in TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e. 2024-11-23T16:14:58,568 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/A/40635fabeefe44d7a3b73195943c2d4b, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/A/8bb3031477224a5eb9dd9cf281e38bc1, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/A/f3cc850d44604796a355f0e2c5c5e0d7] into tmpdir=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/.tmp, totalSize=39.0 K 2024-11-23T16:14:58,568 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37483 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T16:14:58,569 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HStore(1541): 5b34e731541fe76c476f23c5dc5aa28e/B is initiating minor compaction (all files) 2024-11-23T16:14:58,569 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.Compactor(225): Compacting 40635fabeefe44d7a3b73195943c2d4b, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=266, earliestPutTs=1732378493696 2024-11-23T16:14:58,569 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5b34e731541fe76c476f23c5dc5aa28e/B in TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e. 2024-11-23T16:14:58,569 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/B/26a27afba72d4f37b9aeed9a4a861c0a, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/B/e84a9105ca6d42c89fdd7381444e28b0, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/B/79dd470ab1854873b777ea2b1eb91d47] into tmpdir=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/.tmp, totalSize=36.6 K 2024-11-23T16:14:58,569 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.Compactor(225): Compacting 26a27afba72d4f37b9aeed9a4a861c0a, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=266, earliestPutTs=1732378493696 2024-11-23T16:14:58,569 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8bb3031477224a5eb9dd9cf281e38bc1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=285, earliestPutTs=1732378494871 2024-11-23T16:14:58,570 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.Compactor(225): Compacting e84a9105ca6d42c89fdd7381444e28b0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=285, earliestPutTs=1732378494871 2024-11-23T16:14:58,570 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.Compactor(225): Compacting f3cc850d44604796a355f0e2c5c5e0d7, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=302, earliestPutTs=1732378497046 2024-11-23T16:14:58,570 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.Compactor(225): Compacting 79dd470ab1854873b777ea2b1eb91d47, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=302, earliestPutTs=1732378497060 2024-11-23T16:14:58,590 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5b34e731541fe76c476f23c5dc5aa28e#A#compaction#252 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T16:14:58,591 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/.tmp/A/35a792a8c82d4a42ab9e875b5d4415ec is 50, key is test_row_0/A:col10/1732378497063/Put/seqid=0 2024-11-23T16:14:58,615 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5b34e731541fe76c476f23c5dc5aa28e#B#compaction#253 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T16:14:58,615 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/.tmp/B/3e2af7be7931488ab2de3fdc2664dcf2 is 50, key is test_row_0/B:col10/1732378497063/Put/seqid=0 2024-11-23T16:14:58,631 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742122_1298 (size=12983) 2024-11-23T16:14:58,640 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/.tmp/A/35a792a8c82d4a42ab9e875b5d4415ec as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/A/35a792a8c82d4a42ab9e875b5d4415ec 2024-11-23T16:14:58,646 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742123_1299 (size=12983) 2024-11-23T16:14:58,657 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5b34e731541fe76c476f23c5dc5aa28e/A of 5b34e731541fe76c476f23c5dc5aa28e into 35a792a8c82d4a42ab9e875b5d4415ec(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T16:14:58,657 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5b34e731541fe76c476f23c5dc5aa28e: 2024-11-23T16:14:58,657 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., storeName=5b34e731541fe76c476f23c5dc5aa28e/A, priority=13, startTime=1732378498566; duration=0sec 2024-11-23T16:14:58,658 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T16:14:58,658 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5b34e731541fe76c476f23c5dc5aa28e:A 2024-11-23T16:14:58,658 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T16:14:58,660 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37483 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T16:14:58,660 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HStore(1541): 5b34e731541fe76c476f23c5dc5aa28e/C is initiating minor compaction (all files) 2024-11-23T16:14:58,660 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5b34e731541fe76c476f23c5dc5aa28e/C in TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e. 2024-11-23T16:14:58,660 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/C/134baf646ea8406ca000d85dc5aa31bf, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/C/ac1d22e427fe4f85bc8b10a2d854a907, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/C/5c18e55bdda740b793020357fca9c24a] into tmpdir=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/.tmp, totalSize=36.6 K 2024-11-23T16:14:58,660 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.Compactor(225): Compacting 134baf646ea8406ca000d85dc5aa31bf, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=266, earliestPutTs=1732378493696 2024-11-23T16:14:58,661 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.Compactor(225): Compacting ac1d22e427fe4f85bc8b10a2d854a907, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=285, earliestPutTs=1732378494871 2024-11-23T16:14:58,661 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5c18e55bdda740b793020357fca9c24a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=302, earliestPutTs=1732378497060 2024-11-23T16:14:58,689 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5b34e731541fe76c476f23c5dc5aa28e#C#compaction#254 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T16:14:58,689 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/.tmp/C/13c70f3b2e1f44b2bfa8a296f5b6e0f6 is 50, key is test_row_0/C:col10/1732378497063/Put/seqid=0 2024-11-23T16:14:58,725 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742124_1300 (size=12983) 2024-11-23T16:14:58,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(8855): Flush requested on 5b34e731541fe76c476f23c5dc5aa28e 2024-11-23T16:14:58,873 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5b34e731541fe76c476f23c5dc5aa28e 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-23T16:14:58,873 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5b34e731541fe76c476f23c5dc5aa28e, store=A 2024-11-23T16:14:58,873 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-23T16:14:58,873 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5b34e731541fe76c476f23c5dc5aa28e, store=B 2024-11-23T16:14:58,873 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-23T16:14:58,874 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5b34e731541fe76c476f23c5dc5aa28e, store=C 2024-11-23T16:14:58,874 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-23T16:14:58,881 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/.tmp/A/de40026bd26143a199ff77bfe2104826 is 50, key is test_row_0/A:col10/1732378498870/Put/seqid=0 2024-11-23T16:14:58,889 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742125_1301 (size=12301) 2024-11-23T16:14:58,889 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=321 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/.tmp/A/de40026bd26143a199ff77bfe2104826 2024-11-23T16:14:58,915 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:58,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50480 deadline: 1732378558909, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:58,916 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:58,916 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:58,916 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:14:58,921 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:58,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 181 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50424 deadline: 1732378558916, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:58,922 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:58,922 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:58,922 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:14:58,922 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/.tmp/B/a5d4f69e672f49d291b1e64a7207438f is 50, key is test_row_0/B:col10/1732378498870/Put/seqid=0 2024-11-23T16:14:58,941 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742126_1302 (size=12301) 2024-11-23T16:14:58,941 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=321 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/.tmp/B/a5d4f69e672f49d291b1e64a7207438f 2024-11-23T16:14:58,958 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/.tmp/C/2f3b765abef44a36858f58b38e1db70d is 50, key is test_row_0/C:col10/1732378498870/Put/seqid=0 2024-11-23T16:14:58,975 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742127_1303 (size=12301) 2024-11-23T16:14:59,022 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:59,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50480 deadline: 1732378559018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:59,022 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:59,023 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:59,023 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:14:59,031 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:59,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 182 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50424 deadline: 1732378559029, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:59,032 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:59,032 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:59,032 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:14:59,055 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/.tmp/B/3e2af7be7931488ab2de3fdc2664dcf2 as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/B/3e2af7be7931488ab2de3fdc2664dcf2 2024-11-23T16:14:59,062 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5b34e731541fe76c476f23c5dc5aa28e/B of 5b34e731541fe76c476f23c5dc5aa28e into 3e2af7be7931488ab2de3fdc2664dcf2(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T16:14:59,062 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5b34e731541fe76c476f23c5dc5aa28e: 2024-11-23T16:14:59,063 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., storeName=5b34e731541fe76c476f23c5dc5aa28e/B, priority=13, startTime=1732378498567; duration=0sec 2024-11-23T16:14:59,064 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T16:14:59,064 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5b34e731541fe76c476f23c5dc5aa28e:B 2024-11-23T16:14:59,130 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/.tmp/C/13c70f3b2e1f44b2bfa8a296f5b6e0f6 as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/C/13c70f3b2e1f44b2bfa8a296f5b6e0f6 2024-11-23T16:14:59,137 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5b34e731541fe76c476f23c5dc5aa28e/C of 5b34e731541fe76c476f23c5dc5aa28e into 13c70f3b2e1f44b2bfa8a296f5b6e0f6(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T16:14:59,137 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5b34e731541fe76c476f23c5dc5aa28e: 2024-11-23T16:14:59,137 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., storeName=5b34e731541fe76c476f23c5dc5aa28e/C, priority=13, startTime=1732378498567; duration=0sec 2024-11-23T16:14:59,137 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T16:14:59,137 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5b34e731541fe76c476f23c5dc5aa28e:C 2024-11-23T16:14:59,231 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:59,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50480 deadline: 1732378559228, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:59,231 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:59,232 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:59,232 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:14:59,239 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:59,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 183 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50424 deadline: 1732378559238, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:59,239 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:59,239 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:59,239 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:14:59,376 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=321 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/.tmp/C/2f3b765abef44a36858f58b38e1db70d 2024-11-23T16:14:59,381 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/.tmp/A/de40026bd26143a199ff77bfe2104826 as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/A/de40026bd26143a199ff77bfe2104826 2024-11-23T16:14:59,384 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/A/de40026bd26143a199ff77bfe2104826, entries=150, sequenceid=321, filesize=12.0 K 2024-11-23T16:14:59,385 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/.tmp/B/a5d4f69e672f49d291b1e64a7207438f as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/B/a5d4f69e672f49d291b1e64a7207438f 2024-11-23T16:14:59,389 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/B/a5d4f69e672f49d291b1e64a7207438f, entries=150, sequenceid=321, filesize=12.0 K 2024-11-23T16:14:59,389 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/.tmp/C/2f3b765abef44a36858f58b38e1db70d as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/C/2f3b765abef44a36858f58b38e1db70d 2024-11-23T16:14:59,394 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/C/2f3b765abef44a36858f58b38e1db70d, entries=150, sequenceid=321, filesize=12.0 K 2024-11-23T16:14:59,394 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 5b34e731541fe76c476f23c5dc5aa28e in 521ms, sequenceid=321, compaction requested=false 2024-11-23T16:14:59,394 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5b34e731541fe76c476f23c5dc5aa28e: 2024-11-23T16:14:59,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-23T16:14:59,488 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-23T16:14:59,488 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-23T16:14:59,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] procedure2.ProcedureExecutor(1139): Stored pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees 2024-11-23T16:14:59,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-23T16:14:59,491 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-23T16:14:59,491 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-23T16:14:59,491 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=75, ppid=74, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-23T16:14:59,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(8855): Flush requested on 5b34e731541fe76c476f23c5dc5aa28e 2024-11-23T16:14:59,541 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5b34e731541fe76c476f23c5dc5aa28e 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-23T16:14:59,541 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5b34e731541fe76c476f23c5dc5aa28e, store=A 2024-11-23T16:14:59,541 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-23T16:14:59,541 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5b34e731541fe76c476f23c5dc5aa28e, store=B 2024-11-23T16:14:59,541 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-23T16:14:59,541 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5b34e731541fe76c476f23c5dc5aa28e, store=C 2024-11-23T16:14:59,541 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-23T16:14:59,546 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/.tmp/A/d8f342834249487d82c086009393e282 is 50, key is test_row_0/A:col10/1732378498914/Put/seqid=0 2024-11-23T16:14:59,552 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742128_1304 (size=14741) 2024-11-23T16:14:59,582 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:59,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 190 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50424 deadline: 1732378559579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:59,583 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:59,583 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:59,583 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:14:59,585 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:59,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50480 deadline: 1732378559580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:59,586 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:59,586 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:59,586 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:14:59,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-23T16:14:59,644 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-23T16:14:59,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e. 2024-11-23T16:14:59,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e. as already flushing 2024-11-23T16:14:59,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e. 2024-11-23T16:14:59,644 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 5b34e731541fe76c476f23c5dc5aa28e, NAME => 'TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:14:59,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 5b34e731541fe76c476f23c5dc5aa28e, NAME => 'TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:14:59,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5b34e731541fe76c476f23c5dc5aa28e, NAME => 'TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5b34e731541fe76c476f23c5dc5aa28e, NAME => 'TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:14:59,691 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:59,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 191 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50424 deadline: 1732378559688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:59,691 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:59,691 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:59,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50480 deadline: 1732378559688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:59,691 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:59,691 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:14:59,691 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:59,691 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:59,691 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:14:59,797 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-23T16:14:59,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e. 2024-11-23T16:14:59,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e. as already flushing 2024-11-23T16:14:59,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e. 2024-11-23T16:14:59,797 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 5b34e731541fe76c476f23c5dc5aa28e, NAME => 'TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:14:59,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 5b34e731541fe76c476f23c5dc5aa28e, NAME => 'TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:14:59,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5b34e731541fe76c476f23c5dc5aa28e, NAME => 'TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5b34e731541fe76c476f23c5dc5aa28e, NAME => 'TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:14:59,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-23T16:14:59,900 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:59,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 192 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50424 deadline: 1732378559897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:59,900 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:14:59,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50480 deadline: 1732378559898, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:14:59,900 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:59,900 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:59,900 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:14:59,900 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:59,901 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:14:59,901 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:14:59,950 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-23T16:14:59,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e. 2024-11-23T16:14:59,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e. as already flushing 2024-11-23T16:14:59,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e. 2024-11-23T16:14:59,950 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 5b34e731541fe76c476f23c5dc5aa28e, NAME => 'TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:14:59,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 5b34e731541fe76c476f23c5dc5aa28e, NAME => 'TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:14:59,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5b34e731541fe76c476f23c5dc5aa28e, NAME => 'TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5b34e731541fe76c476f23c5dc5aa28e, NAME => 'TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:14:59,953 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=338 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/.tmp/A/d8f342834249487d82c086009393e282 2024-11-23T16:14:59,960 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/.tmp/B/36b107526f9b4e619ace926d9c4b9735 is 50, key is test_row_0/B:col10/1732378498914/Put/seqid=0 2024-11-23T16:14:59,964 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742129_1305 (size=12301) 2024-11-23T16:15:00,103 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-23T16:15:00,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e. 2024-11-23T16:15:00,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e. as already flushing 2024-11-23T16:15:00,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e. 2024-11-23T16:15:00,103 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 5b34e731541fe76c476f23c5dc5aa28e, NAME => 'TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:15:00,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 5b34e731541fe76c476f23c5dc5aa28e, NAME => 'TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:15:00,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5b34e731541fe76c476f23c5dc5aa28e, NAME => 'TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5b34e731541fe76c476f23c5dc5aa28e, NAME => 'TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:15:00,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-23T16:15:00,210 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:00,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 193 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50424 deadline: 1732378560208, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:00,211 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:00,211 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:00,211 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:00,211 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:15:00,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50480 deadline: 1732378560208, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:00,211 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:00,211 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:00,211 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:15:00,256 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-23T16:15:00,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e. 2024-11-23T16:15:00,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e. as already flushing 2024-11-23T16:15:00,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e. 2024-11-23T16:15:00,256 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 5b34e731541fe76c476f23c5dc5aa28e, NAME => 'TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:15:00,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 5b34e731541fe76c476f23c5dc5aa28e, NAME => 'TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:15:00,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5b34e731541fe76c476f23c5dc5aa28e, NAME => 'TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5b34e731541fe76c476f23c5dc5aa28e, NAME => 'TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:15:00,365 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=338 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/.tmp/B/36b107526f9b4e619ace926d9c4b9735 2024-11-23T16:15:00,372 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/.tmp/C/fb5c30bfa8884523962e7df926400124 is 50, key is test_row_0/C:col10/1732378498914/Put/seqid=0 2024-11-23T16:15:00,376 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742130_1306 (size=12301) 2024-11-23T16:15:00,409 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-23T16:15:00,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e. 2024-11-23T16:15:00,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e. as already flushing 2024-11-23T16:15:00,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e. 2024-11-23T16:15:00,409 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 5b34e731541fe76c476f23c5dc5aa28e, NAME => 'TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:15:00,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 5b34e731541fe76c476f23c5dc5aa28e, NAME => 'TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:15:00,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5b34e731541fe76c476f23c5dc5aa28e, NAME => 'TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5b34e731541fe76c476f23c5dc5aa28e, NAME => 'TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:15:00,562 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-23T16:15:00,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e. 2024-11-23T16:15:00,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e. as already flushing 2024-11-23T16:15:00,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e. 2024-11-23T16:15:00,563 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 5b34e731541fe76c476f23c5dc5aa28e, NAME => 'TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:15:00,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 5b34e731541fe76c476f23c5dc5aa28e, NAME => 'TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:15:00,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5b34e731541fe76c476f23c5dc5aa28e, NAME => 'TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5b34e731541fe76c476f23c5dc5aa28e, NAME => 'TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:15:00,588 INFO [Thread-1122 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1122. 2024-11-23T16:15:00,589 DEBUG [Thread-1122 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-23T16:15:00,589 DEBUG [Thread-1122 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T16:15:00,589 DEBUG [Thread-1122 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T16:15:00,589 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-23T16:15:00,590 INFO [Thread-1126 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1126. 2024-11-23T16:15:00,590 DEBUG [Thread-1126 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-23T16:15:00,590 DEBUG [Thread-1126 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T16:15:00,590 DEBUG [Thread-1126 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T16:15:00,590 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-23T16:15:00,591 INFO [Thread-1124 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1124. 2024-11-23T16:15:00,591 DEBUG [Thread-1124 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-23T16:15:00,591 DEBUG [Thread-1124 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T16:15:00,591 DEBUG [Thread-1124 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T16:15:00,591 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-23T16:15:00,593 INFO [Thread-1120 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1120. 2024-11-23T16:15:00,593 DEBUG [Thread-1120 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-23T16:15:00,593 DEBUG [Thread-1120 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T16:15:00,593 DEBUG [Thread-1120 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T16:15:00,593 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-23T16:15:00,594 INFO [Thread-1118 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1118. 2024-11-23T16:15:00,594 DEBUG [Thread-1118 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-23T16:15:00,594 DEBUG [Thread-1118 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T16:15:00,594 DEBUG [Thread-1118 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T16:15:00,594 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-23T16:15:00,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-23T16:15:00,715 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-23T16:15:00,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e. 2024-11-23T16:15:00,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e. as already flushing 2024-11-23T16:15:00,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e. 2024-11-23T16:15:00,716 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 5b34e731541fe76c476f23c5dc5aa28e, NAME => 'TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:15:00,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 5b34e731541fe76c476f23c5dc5aa28e, NAME => 'TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:15:00,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5b34e731541fe76c476f23c5dc5aa28e, NAME => 'TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5b34e731541fe76c476f23c5dc5aa28e, NAME => 'TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:15:00,718 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:00,718 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:00,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50480 deadline: 1732378560718, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:00,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 194 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50424 deadline: 1732378560718, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:00,719 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:00,719 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:00,719 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:00,719 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:00,719 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:15:00,719 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:15:00,777 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=338 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/.tmp/C/fb5c30bfa8884523962e7df926400124 2024-11-23T16:15:00,780 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/.tmp/A/d8f342834249487d82c086009393e282 as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/A/d8f342834249487d82c086009393e282 2024-11-23T16:15:00,784 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/A/d8f342834249487d82c086009393e282, entries=200, sequenceid=338, filesize=14.4 K 2024-11-23T16:15:00,784 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/.tmp/B/36b107526f9b4e619ace926d9c4b9735 as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/B/36b107526f9b4e619ace926d9c4b9735 2024-11-23T16:15:00,787 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/B/36b107526f9b4e619ace926d9c4b9735, entries=150, sequenceid=338, filesize=12.0 K 2024-11-23T16:15:00,788 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/.tmp/C/fb5c30bfa8884523962e7df926400124 as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/C/fb5c30bfa8884523962e7df926400124 2024-11-23T16:15:00,791 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/C/fb5c30bfa8884523962e7df926400124, entries=150, sequenceid=338, filesize=12.0 K 2024-11-23T16:15:00,791 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 5b34e731541fe76c476f23c5dc5aa28e in 1250ms, sequenceid=338, compaction requested=true 2024-11-23T16:15:00,791 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5b34e731541fe76c476f23c5dc5aa28e: 2024-11-23T16:15:00,791 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5b34e731541fe76c476f23c5dc5aa28e:A, priority=-2147483648, current under compaction store size is 1 2024-11-23T16:15:00,791 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T16:15:00,791 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T16:15:00,791 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5b34e731541fe76c476f23c5dc5aa28e:B, priority=-2147483648, current under compaction store size is 2 2024-11-23T16:15:00,791 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T16:15:00,791 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5b34e731541fe76c476f23c5dc5aa28e:C, priority=-2147483648, current under compaction store size is 3 2024-11-23T16:15:00,792 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T16:15:00,792 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T16:15:00,792 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40025 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T16:15:00,792 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T16:15:00,792 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HStore(1541): 5b34e731541fe76c476f23c5dc5aa28e/A is initiating minor compaction (all files) 2024-11-23T16:15:00,792 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HStore(1541): 5b34e731541fe76c476f23c5dc5aa28e/B is initiating minor compaction (all files) 2024-11-23T16:15:00,792 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5b34e731541fe76c476f23c5dc5aa28e/A in TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e. 2024-11-23T16:15:00,792 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5b34e731541fe76c476f23c5dc5aa28e/B in TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e. 2024-11-23T16:15:00,792 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/A/35a792a8c82d4a42ab9e875b5d4415ec, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/A/de40026bd26143a199ff77bfe2104826, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/A/d8f342834249487d82c086009393e282] into tmpdir=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/.tmp, totalSize=39.1 K 2024-11-23T16:15:00,792 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/B/3e2af7be7931488ab2de3fdc2664dcf2, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/B/a5d4f69e672f49d291b1e64a7207438f, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/B/36b107526f9b4e619ace926d9c4b9735] into tmpdir=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/.tmp, totalSize=36.7 K 2024-11-23T16:15:00,793 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.Compactor(225): Compacting 35a792a8c82d4a42ab9e875b5d4415ec, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=302, earliestPutTs=1732378497060 2024-11-23T16:15:00,793 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.Compactor(225): Compacting 3e2af7be7931488ab2de3fdc2664dcf2, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=302, earliestPutTs=1732378497060 2024-11-23T16:15:00,793 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.Compactor(225): Compacting de40026bd26143a199ff77bfe2104826, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=321, earliestPutTs=1732378497724 2024-11-23T16:15:00,793 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.Compactor(225): Compacting a5d4f69e672f49d291b1e64a7207438f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=321, earliestPutTs=1732378497724 2024-11-23T16:15:00,793 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.Compactor(225): Compacting d8f342834249487d82c086009393e282, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=338, earliestPutTs=1732378498886 2024-11-23T16:15:00,793 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.Compactor(225): Compacting 36b107526f9b4e619ace926d9c4b9735, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=338, earliestPutTs=1732378498886 2024-11-23T16:15:00,799 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5b34e731541fe76c476f23c5dc5aa28e#B#compaction#261 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-23T16:15:00,799 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5b34e731541fe76c476f23c5dc5aa28e#A#compaction#262 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T16:15:00,800 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/.tmp/A/f02dece50b324b9dabd33db13263eb5a is 50, key is test_row_0/A:col10/1732378498914/Put/seqid=0 2024-11-23T16:15:00,800 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/.tmp/B/67042741dff945388c2a726ac6428f4e is 50, key is test_row_0/B:col10/1732378498914/Put/seqid=0 2024-11-23T16:15:00,803 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742132_1308 (size=13085) 2024-11-23T16:15:00,803 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742131_1307 (size=13085) 2024-11-23T16:15:00,807 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/.tmp/A/f02dece50b324b9dabd33db13263eb5a as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/A/f02dece50b324b9dabd33db13263eb5a 2024-11-23T16:15:00,811 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5b34e731541fe76c476f23c5dc5aa28e/A of 5b34e731541fe76c476f23c5dc5aa28e into f02dece50b324b9dabd33db13263eb5a(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T16:15:00,811 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5b34e731541fe76c476f23c5dc5aa28e: 2024-11-23T16:15:00,811 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., storeName=5b34e731541fe76c476f23c5dc5aa28e/A, priority=13, startTime=1732378500791; duration=0sec 2024-11-23T16:15:00,811 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T16:15:00,811 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5b34e731541fe76c476f23c5dc5aa28e:A 2024-11-23T16:15:00,811 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T16:15:00,811 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T16:15:00,811 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HStore(1541): 5b34e731541fe76c476f23c5dc5aa28e/C is initiating minor compaction (all files) 2024-11-23T16:15:00,811 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5b34e731541fe76c476f23c5dc5aa28e/C in TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e. 2024-11-23T16:15:00,812 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/C/13c70f3b2e1f44b2bfa8a296f5b6e0f6, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/C/2f3b765abef44a36858f58b38e1db70d, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/C/fb5c30bfa8884523962e7df926400124] into tmpdir=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/.tmp, totalSize=36.7 K 2024-11-23T16:15:00,812 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.Compactor(225): Compacting 13c70f3b2e1f44b2bfa8a296f5b6e0f6, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=302, earliestPutTs=1732378497060 2024-11-23T16:15:00,812 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2f3b765abef44a36858f58b38e1db70d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=321, earliestPutTs=1732378497724 2024-11-23T16:15:00,812 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.Compactor(225): Compacting fb5c30bfa8884523962e7df926400124, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=338, earliestPutTs=1732378498886 2024-11-23T16:15:00,818 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5b34e731541fe76c476f23c5dc5aa28e#C#compaction#263 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T16:15:00,818 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/.tmp/C/df34411f41b544b5a8323f37cadbf52e is 50, key is test_row_0/C:col10/1732378498914/Put/seqid=0 2024-11-23T16:15:00,821 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742133_1309 (size=13085) 2024-11-23T16:15:00,869 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-23T16:15:00,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e. 2024-11-23T16:15:00,869 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2902): Flushing 5b34e731541fe76c476f23c5dc5aa28e 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-23T16:15:00,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5b34e731541fe76c476f23c5dc5aa28e, store=A 2024-11-23T16:15:00,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-23T16:15:00,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5b34e731541fe76c476f23c5dc5aa28e, store=B 2024-11-23T16:15:00,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-23T16:15:00,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5b34e731541fe76c476f23c5dc5aa28e, store=C 2024-11-23T16:15:00,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-23T16:15:00,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/.tmp/A/b16d64b54fb24938aba13b0ae2594b04 is 50, key is test_row_0/A:col10/1732378499577/Put/seqid=0 2024-11-23T16:15:00,876 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742134_1310 (size=12301) 2024-11-23T16:15:01,208 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/.tmp/B/67042741dff945388c2a726ac6428f4e as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/B/67042741dff945388c2a726ac6428f4e 2024-11-23T16:15:01,212 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5b34e731541fe76c476f23c5dc5aa28e/B of 5b34e731541fe76c476f23c5dc5aa28e into 67042741dff945388c2a726ac6428f4e(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T16:15:01,212 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5b34e731541fe76c476f23c5dc5aa28e: 2024-11-23T16:15:01,212 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., storeName=5b34e731541fe76c476f23c5dc5aa28e/B, priority=13, startTime=1732378500791; duration=0sec 2024-11-23T16:15:01,212 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T16:15:01,212 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5b34e731541fe76c476f23c5dc5aa28e:B 2024-11-23T16:15:01,225 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/.tmp/C/df34411f41b544b5a8323f37cadbf52e as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/C/df34411f41b544b5a8323f37cadbf52e 2024-11-23T16:15:01,228 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5b34e731541fe76c476f23c5dc5aa28e/C of 5b34e731541fe76c476f23c5dc5aa28e into df34411f41b544b5a8323f37cadbf52e(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T16:15:01,229 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5b34e731541fe76c476f23c5dc5aa28e: 2024-11-23T16:15:01,229 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., storeName=5b34e731541fe76c476f23c5dc5aa28e/C, priority=13, startTime=1732378500791; duration=0sec 2024-11-23T16:15:01,229 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T16:15:01,229 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5b34e731541fe76c476f23c5dc5aa28e:C 2024-11-23T16:15:01,277 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=356 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/.tmp/A/b16d64b54fb24938aba13b0ae2594b04 2024-11-23T16:15:01,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/.tmp/B/81da43f44a7e4d44b7caddf8bf28dd9d is 50, key is test_row_0/B:col10/1732378499577/Put/seqid=0 2024-11-23T16:15:01,287 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742135_1311 (size=12301) 2024-11-23T16:15:01,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-23T16:15:01,688 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=356 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/.tmp/B/81da43f44a7e4d44b7caddf8bf28dd9d 2024-11-23T16:15:01,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/.tmp/C/7fac4a050a104da1ad2136ef7dfdb5e7 is 50, key is test_row_0/C:col10/1732378499577/Put/seqid=0 2024-11-23T16:15:01,698 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742136_1312 (size=12301) 2024-11-23T16:15:01,698 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=356 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/.tmp/C/7fac4a050a104da1ad2136ef7dfdb5e7 2024-11-23T16:15:01,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/.tmp/A/b16d64b54fb24938aba13b0ae2594b04 as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/A/b16d64b54fb24938aba13b0ae2594b04 2024-11-23T16:15:01,706 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/A/b16d64b54fb24938aba13b0ae2594b04, entries=150, sequenceid=356, filesize=12.0 K 2024-11-23T16:15:01,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/.tmp/B/81da43f44a7e4d44b7caddf8bf28dd9d as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/B/81da43f44a7e4d44b7caddf8bf28dd9d 2024-11-23T16:15:01,710 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/B/81da43f44a7e4d44b7caddf8bf28dd9d, entries=150, sequenceid=356, filesize=12.0 K 2024-11-23T16:15:01,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/.tmp/C/7fac4a050a104da1ad2136ef7dfdb5e7 as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/C/7fac4a050a104da1ad2136ef7dfdb5e7 2024-11-23T16:15:01,714 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/C/7fac4a050a104da1ad2136ef7dfdb5e7, entries=150, sequenceid=356, filesize=12.0 K 2024-11-23T16:15:01,715 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=0 B/0 for 5b34e731541fe76c476f23c5dc5aa28e in 846ms, sequenceid=356, compaction requested=false 2024-11-23T16:15:01,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2603): Flush status journal for 5b34e731541fe76c476f23c5dc5aa28e: 2024-11-23T16:15:01,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e. 2024-11-23T16:15:01,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=75 2024-11-23T16:15:01,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.HMaster(4169): Remote procedure done, pid=75 2024-11-23T16:15:01,718 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=75, resume processing ppid=74 2024-11-23T16:15:01,718 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=75, ppid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2250 sec 2024-11-23T16:15:01,719 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees in 2.2290 sec 2024-11-23T16:15:01,730 INFO [Thread-1115 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1115. 2024-11-23T16:15:01,730 DEBUG [Thread-1115 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-23T16:15:01,730 DEBUG [Thread-1115 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T16:15:01,730 DEBUG [Thread-1115 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T16:15:01,730 INFO [Thread-1111 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1111. 2024-11-23T16:15:01,730 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-23T16:15:01,730 DEBUG [Thread-1111 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-23T16:15:01,730 DEBUG [Thread-1111 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T16:15:01,730 DEBUG [Thread-1111 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T16:15:01,730 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-23T16:15:03,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-23T16:15:03,658 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-23T16:15:07,180 INFO [Thread-1113 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1113. 2024-11-23T16:15:07,180 DEBUG [Thread-1113 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-23T16:15:07,180 DEBUG [Thread-1113 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T16:15:07,180 DEBUG [Thread-1113 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T16:15:07,180 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-23T16:15:07,189 INFO [Thread-1109 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1109. 2024-11-23T16:15:07,189 DEBUG [Thread-1109 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-23T16:15:07,189 DEBUG [Thread-1109 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T16:15:07,189 DEBUG [Thread-1109 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T16:15:07,189 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-23T16:15:07,209 INFO [Thread-1107 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1107. 2024-11-23T16:15:07,209 DEBUG [Thread-1107 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-23T16:15:07,209 DEBUG [Thread-1107 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T16:15:07,209 DEBUG [Thread-1107 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T16:15:07,209 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-23T16:15:07,209 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-23T16:15:07,209 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 23 2024-11-23T16:15:07,209 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 14 2024-11-23T16:15:07,209 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 132 2024-11-23T16:15:07,209 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 10 2024-11-23T16:15:07,209 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 108 2024-11-23T16:15:07,209 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-23T16:15:07,209 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2946 2024-11-23T16:15:07,209 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2912 2024-11-23T16:15:07,209 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2877 2024-11-23T16:15:07,209 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2958 2024-11-23T16:15:07,209 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2902 2024-11-23T16:15:07,209 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-23T16:15:07,210 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-23T16:15:07,210 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testGetAtomicity(AcidGuaranteesTestBase.java:107) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-23T16:15:07,210 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T16:15:07,210 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T16:15:07,210 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-23T16:15:07,210 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-23T16:15:07,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] procedure2.ProcedureExecutor(1139): Stored pid=76, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-23T16:15:07,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-23T16:15:07,213 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732378507213"}]},"ts":"1732378507213"} 2024-11-23T16:15:07,214 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-23T16:15:07,214 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-23T16:15:07,215 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=77, ppid=76, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-23T16:15:07,216 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=78, ppid=77, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5b34e731541fe76c476f23c5dc5aa28e, UNASSIGN}] 2024-11-23T16:15:07,217 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=78, ppid=77, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5b34e731541fe76c476f23c5dc5aa28e, UNASSIGN 2024-11-23T16:15:07,217 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=78 updating hbase:meta row=5b34e731541fe76c476f23c5dc5aa28e, regionState=CLOSING, regionLocation=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:07,218 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=78, ppid=77, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=5b34e731541fe76c476f23c5dc5aa28e, UNASSIGN because future has completed 2024-11-23T16:15:07,218 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-23T16:15:07,218 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=79, ppid=78, state=RUNNABLE, hasLock=false; CloseRegionProcedure 5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375}] 2024-11-23T16:15:07,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-23T16:15:07,370 INFO [RS_CLOSE_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] handler.UnassignRegionHandler(122): Close 5b34e731541fe76c476f23c5dc5aa28e 2024-11-23T16:15:07,370 DEBUG [RS_CLOSE_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-23T16:15:07,370 DEBUG [RS_CLOSE_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.HRegion(1722): Closing 5b34e731541fe76c476f23c5dc5aa28e, disabling compactions & flushes 2024-11-23T16:15:07,371 INFO [RS_CLOSE_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e. 2024-11-23T16:15:07,371 DEBUG [RS_CLOSE_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e. 2024-11-23T16:15:07,371 DEBUG [RS_CLOSE_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e. after waiting 0 ms 2024-11-23T16:15:07,371 DEBUG [RS_CLOSE_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e. 2024-11-23T16:15:07,371 INFO [RS_CLOSE_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.HRegion(2902): Flushing 5b34e731541fe76c476f23c5dc5aa28e 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-23T16:15:07,371 DEBUG [RS_CLOSE_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5b34e731541fe76c476f23c5dc5aa28e, store=A 2024-11-23T16:15:07,371 DEBUG [RS_CLOSE_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-23T16:15:07,371 DEBUG [RS_CLOSE_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5b34e731541fe76c476f23c5dc5aa28e, store=B 2024-11-23T16:15:07,371 DEBUG [RS_CLOSE_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-23T16:15:07,371 DEBUG [RS_CLOSE_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5b34e731541fe76c476f23c5dc5aa28e, store=C 2024-11-23T16:15:07,371 DEBUG [RS_CLOSE_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-23T16:15:07,375 DEBUG [RS_CLOSE_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/.tmp/A/d23a200809414ae9ab2c443e5d86aa47 is 50, key is test_row_0/A:col10/1732378507179/Put/seqid=0 2024-11-23T16:15:07,378 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742137_1313 (size=9857) 2024-11-23T16:15:07,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-23T16:15:07,779 INFO [RS_CLOSE_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=366 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/.tmp/A/d23a200809414ae9ab2c443e5d86aa47 2024-11-23T16:15:07,785 DEBUG [RS_CLOSE_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/.tmp/B/0b07f17c24394a589819a6bc19ae5a34 is 50, key is test_row_0/B:col10/1732378507179/Put/seqid=0 2024-11-23T16:15:07,788 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742138_1314 (size=9857) 2024-11-23T16:15:07,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-23T16:15:08,189 INFO [RS_CLOSE_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=366 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/.tmp/B/0b07f17c24394a589819a6bc19ae5a34 2024-11-23T16:15:08,195 DEBUG [RS_CLOSE_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/.tmp/C/8ffabef8bade4d6a9c366283fda636c4 is 50, key is test_row_0/C:col10/1732378507179/Put/seqid=0 2024-11-23T16:15:08,199 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742139_1315 (size=9857) 2024-11-23T16:15:08,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-23T16:15:08,600 INFO [RS_CLOSE_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=366 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/.tmp/C/8ffabef8bade4d6a9c366283fda636c4 2024-11-23T16:15:08,604 DEBUG [RS_CLOSE_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/.tmp/A/d23a200809414ae9ab2c443e5d86aa47 as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/A/d23a200809414ae9ab2c443e5d86aa47 2024-11-23T16:15:08,607 INFO [RS_CLOSE_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/A/d23a200809414ae9ab2c443e5d86aa47, entries=100, sequenceid=366, filesize=9.6 K 2024-11-23T16:15:08,608 DEBUG [RS_CLOSE_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/.tmp/B/0b07f17c24394a589819a6bc19ae5a34 as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/B/0b07f17c24394a589819a6bc19ae5a34 2024-11-23T16:15:08,611 INFO [RS_CLOSE_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/B/0b07f17c24394a589819a6bc19ae5a34, entries=100, sequenceid=366, filesize=9.6 K 2024-11-23T16:15:08,612 DEBUG [RS_CLOSE_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/.tmp/C/8ffabef8bade4d6a9c366283fda636c4 as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/C/8ffabef8bade4d6a9c366283fda636c4 2024-11-23T16:15:08,615 INFO [RS_CLOSE_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/C/8ffabef8bade4d6a9c366283fda636c4, entries=100, sequenceid=366, filesize=9.6 K 2024-11-23T16:15:08,616 INFO [RS_CLOSE_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 5b34e731541fe76c476f23c5dc5aa28e in 1245ms, sequenceid=366, compaction requested=true 2024-11-23T16:15:08,616 DEBUG [StoreCloser-TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/A/cc582adb0e4a4cc6b18911da9d4817aa, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/A/cf65c7a4c4994b898e5df62963e55b3e, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/A/6098615294af43fb8b64b558f9dec825, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/A/6c8c761768e0445cb6186523cccc5209, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/A/95d5b137d6e14adf80aa0c4f06f38564, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/A/bf74df63ad144d31ac9ea6914ff95f03, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/A/f56d4b34d42449eba8c6c7be0ab73aa0, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/A/5384f824458046c6a6eed0566315ab50, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/A/3879c73ea540499487e8660ac615f037, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/A/061f677f613e494fa09bff323886b641, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/A/46b14146244e4c5b85d34e3e83c18b9d, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/A/dae109f61f604db98904191b015c8720, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/A/adea4d701c414fb38fe3cb3cc61e0a4b, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/A/e0c94f0e49ff4b56890f7c49bab74512, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/A/eb4498eb5bb84238ae869bcfaf52f8d9, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/A/7a25f247685b4423afcc6d516b6f0c46, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/A/40635fabeefe44d7a3b73195943c2d4b, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/A/b15e6612f7c347b98df70d673799702e, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/A/8bb3031477224a5eb9dd9cf281e38bc1, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/A/f3cc850d44604796a355f0e2c5c5e0d7, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/A/35a792a8c82d4a42ab9e875b5d4415ec, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/A/de40026bd26143a199ff77bfe2104826, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/A/d8f342834249487d82c086009393e282] to archive 2024-11-23T16:15:08,617 DEBUG [StoreCloser-TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-23T16:15:08,619 DEBUG [StoreCloser-TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/A/cc582adb0e4a4cc6b18911da9d4817aa to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/A/cc582adb0e4a4cc6b18911da9d4817aa 2024-11-23T16:15:08,620 DEBUG [StoreCloser-TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/A/cf65c7a4c4994b898e5df62963e55b3e to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/A/cf65c7a4c4994b898e5df62963e55b3e 2024-11-23T16:15:08,621 DEBUG [StoreCloser-TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/A/6098615294af43fb8b64b558f9dec825 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/A/6098615294af43fb8b64b558f9dec825 2024-11-23T16:15:08,622 DEBUG [StoreCloser-TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/A/6c8c761768e0445cb6186523cccc5209 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/A/6c8c761768e0445cb6186523cccc5209 2024-11-23T16:15:08,622 DEBUG [StoreCloser-TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/A/95d5b137d6e14adf80aa0c4f06f38564 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/A/95d5b137d6e14adf80aa0c4f06f38564 2024-11-23T16:15:08,624 DEBUG [StoreCloser-TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/A/bf74df63ad144d31ac9ea6914ff95f03 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/A/bf74df63ad144d31ac9ea6914ff95f03 2024-11-23T16:15:08,625 DEBUG [StoreCloser-TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/A/f56d4b34d42449eba8c6c7be0ab73aa0 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/A/f56d4b34d42449eba8c6c7be0ab73aa0 2024-11-23T16:15:08,626 DEBUG [StoreCloser-TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/A/5384f824458046c6a6eed0566315ab50 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/A/5384f824458046c6a6eed0566315ab50 2024-11-23T16:15:08,626 DEBUG [StoreCloser-TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/A/3879c73ea540499487e8660ac615f037 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/A/3879c73ea540499487e8660ac615f037 2024-11-23T16:15:08,627 DEBUG [StoreCloser-TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/A/061f677f613e494fa09bff323886b641 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/A/061f677f613e494fa09bff323886b641 2024-11-23T16:15:08,628 DEBUG [StoreCloser-TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/A/46b14146244e4c5b85d34e3e83c18b9d to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/A/46b14146244e4c5b85d34e3e83c18b9d 2024-11-23T16:15:08,629 DEBUG [StoreCloser-TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/A/dae109f61f604db98904191b015c8720 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/A/dae109f61f604db98904191b015c8720 2024-11-23T16:15:08,630 DEBUG [StoreCloser-TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/A/adea4d701c414fb38fe3cb3cc61e0a4b to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/A/adea4d701c414fb38fe3cb3cc61e0a4b 2024-11-23T16:15:08,631 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-23T16:15:08,631 DEBUG [StoreCloser-TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/A/e0c94f0e49ff4b56890f7c49bab74512 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/A/e0c94f0e49ff4b56890f7c49bab74512 2024-11-23T16:15:08,632 DEBUG [StoreCloser-TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/A/eb4498eb5bb84238ae869bcfaf52f8d9 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/A/eb4498eb5bb84238ae869bcfaf52f8d9 2024-11-23T16:15:08,633 DEBUG [StoreCloser-TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/A/7a25f247685b4423afcc6d516b6f0c46 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/A/7a25f247685b4423afcc6d516b6f0c46 2024-11-23T16:15:08,634 DEBUG [StoreCloser-TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/A/40635fabeefe44d7a3b73195943c2d4b to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/A/40635fabeefe44d7a3b73195943c2d4b 2024-11-23T16:15:08,635 DEBUG [StoreCloser-TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/A/b15e6612f7c347b98df70d673799702e to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/A/b15e6612f7c347b98df70d673799702e 2024-11-23T16:15:08,636 DEBUG [StoreCloser-TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/A/8bb3031477224a5eb9dd9cf281e38bc1 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/A/8bb3031477224a5eb9dd9cf281e38bc1 2024-11-23T16:15:08,637 DEBUG [StoreCloser-TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/A/f3cc850d44604796a355f0e2c5c5e0d7 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/A/f3cc850d44604796a355f0e2c5c5e0d7 2024-11-23T16:15:08,638 DEBUG [StoreCloser-TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/A/35a792a8c82d4a42ab9e875b5d4415ec to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/A/35a792a8c82d4a42ab9e875b5d4415ec 2024-11-23T16:15:08,638 DEBUG [StoreCloser-TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/A/de40026bd26143a199ff77bfe2104826 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/A/de40026bd26143a199ff77bfe2104826 2024-11-23T16:15:08,639 DEBUG [StoreCloser-TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/A/d8f342834249487d82c086009393e282 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/A/d8f342834249487d82c086009393e282 2024-11-23T16:15:08,641 DEBUG [StoreCloser-TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/B/09a9483ab01c4a93b5dd72f29e8976fe, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/B/fbb105af53f64607977e5eb3818f7143, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/B/beeba66974a54d2cb38347f72f7c20a9, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/B/f3c03781921d4b4eb47abceedfcbf825, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/B/a5c2b481b5434f13aa1931b71ef76079, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/B/7516c502ee024ee0b7fee15538b1d040, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/B/c1896e0d029b4e55b9718a155f980ad2, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/B/a1359f923efa46d196a5e465dd7a1df1, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/B/407996804dcb4a70841312a6eec170bd, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/B/4b1059e84c3c49acb94d51939ca6df50, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/B/d708b428fb784ae9b2fd4aed7439b615, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/B/0027e9b63c7b41239c9f6219eac43145, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/B/7677c1f3a1e34a458213645d1662ec7b, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/B/329fd3cf8b0f4223a7665f8704a3ea99, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/B/77da15054a8849ee8605755feea9cc58, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/B/31dba82a738d4286be490e41d057aefc, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/B/26a27afba72d4f37b9aeed9a4a861c0a, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/B/3d4ef229f7684d3bb4820557740d2398, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/B/e84a9105ca6d42c89fdd7381444e28b0, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/B/3e2af7be7931488ab2de3fdc2664dcf2, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/B/79dd470ab1854873b777ea2b1eb91d47, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/B/a5d4f69e672f49d291b1e64a7207438f, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/B/36b107526f9b4e619ace926d9c4b9735] to archive 2024-11-23T16:15:08,641 DEBUG [StoreCloser-TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-23T16:15:08,643 DEBUG [StoreCloser-TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/B/09a9483ab01c4a93b5dd72f29e8976fe to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/B/09a9483ab01c4a93b5dd72f29e8976fe 2024-11-23T16:15:08,644 DEBUG [StoreCloser-TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/B/fbb105af53f64607977e5eb3818f7143 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/B/fbb105af53f64607977e5eb3818f7143 2024-11-23T16:15:08,645 DEBUG [StoreCloser-TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/B/beeba66974a54d2cb38347f72f7c20a9 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/B/beeba66974a54d2cb38347f72f7c20a9 2024-11-23T16:15:08,645 DEBUG [StoreCloser-TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/B/f3c03781921d4b4eb47abceedfcbf825 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/B/f3c03781921d4b4eb47abceedfcbf825 2024-11-23T16:15:08,646 DEBUG [StoreCloser-TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/B/a5c2b481b5434f13aa1931b71ef76079 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/B/a5c2b481b5434f13aa1931b71ef76079 2024-11-23T16:15:08,647 DEBUG [StoreCloser-TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/B/7516c502ee024ee0b7fee15538b1d040 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/B/7516c502ee024ee0b7fee15538b1d040 2024-11-23T16:15:08,648 DEBUG [StoreCloser-TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/B/c1896e0d029b4e55b9718a155f980ad2 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/B/c1896e0d029b4e55b9718a155f980ad2 2024-11-23T16:15:08,648 DEBUG [StoreCloser-TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/B/a1359f923efa46d196a5e465dd7a1df1 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/B/a1359f923efa46d196a5e465dd7a1df1 2024-11-23T16:15:08,649 DEBUG [StoreCloser-TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/B/407996804dcb4a70841312a6eec170bd to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/B/407996804dcb4a70841312a6eec170bd 2024-11-23T16:15:08,650 DEBUG [StoreCloser-TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/B/4b1059e84c3c49acb94d51939ca6df50 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/B/4b1059e84c3c49acb94d51939ca6df50 2024-11-23T16:15:08,651 DEBUG [StoreCloser-TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/B/d708b428fb784ae9b2fd4aed7439b615 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/B/d708b428fb784ae9b2fd4aed7439b615 2024-11-23T16:15:08,652 DEBUG [StoreCloser-TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/B/0027e9b63c7b41239c9f6219eac43145 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/B/0027e9b63c7b41239c9f6219eac43145 2024-11-23T16:15:08,652 DEBUG [StoreCloser-TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/B/7677c1f3a1e34a458213645d1662ec7b to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/B/7677c1f3a1e34a458213645d1662ec7b 2024-11-23T16:15:08,653 DEBUG [StoreCloser-TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/B/329fd3cf8b0f4223a7665f8704a3ea99 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/B/329fd3cf8b0f4223a7665f8704a3ea99 2024-11-23T16:15:08,654 DEBUG [StoreCloser-TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/B/77da15054a8849ee8605755feea9cc58 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/B/77da15054a8849ee8605755feea9cc58 2024-11-23T16:15:08,655 DEBUG [StoreCloser-TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/B/31dba82a738d4286be490e41d057aefc to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/B/31dba82a738d4286be490e41d057aefc 2024-11-23T16:15:08,656 DEBUG [StoreCloser-TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/B/26a27afba72d4f37b9aeed9a4a861c0a to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/B/26a27afba72d4f37b9aeed9a4a861c0a 2024-11-23T16:15:08,656 DEBUG [StoreCloser-TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/B/3d4ef229f7684d3bb4820557740d2398 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/B/3d4ef229f7684d3bb4820557740d2398 2024-11-23T16:15:08,657 DEBUG [StoreCloser-TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/B/e84a9105ca6d42c89fdd7381444e28b0 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/B/e84a9105ca6d42c89fdd7381444e28b0 2024-11-23T16:15:08,659 DEBUG [StoreCloser-TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/B/3e2af7be7931488ab2de3fdc2664dcf2 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/B/3e2af7be7931488ab2de3fdc2664dcf2 2024-11-23T16:15:08,660 DEBUG [StoreCloser-TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/B/79dd470ab1854873b777ea2b1eb91d47 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/B/79dd470ab1854873b777ea2b1eb91d47 2024-11-23T16:15:08,661 DEBUG [StoreCloser-TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/B/a5d4f69e672f49d291b1e64a7207438f to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/B/a5d4f69e672f49d291b1e64a7207438f 2024-11-23T16:15:08,662 DEBUG [StoreCloser-TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/B/36b107526f9b4e619ace926d9c4b9735 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/B/36b107526f9b4e619ace926d9c4b9735 2024-11-23T16:15:08,663 DEBUG [StoreCloser-TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/C/011c9d7b98614d99b81fab8639fdf62d, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/C/22c41e52183c4f37949452f6e1234d84, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/C/cb95b6a55dd14f97986641f953711dc6, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/C/e131724238234ebca997fd18293eabfe, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/C/0ec7a896d69c4ca1a2ab2571709796b6, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/C/6aa31c68f85f450891c718706a92e59c, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/C/367246ca68e04b1fb74befd857be030e, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/C/ff1f7c573de546d893d1db073b59e412, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/C/a6bb103255e34ed98332935e32943a73, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/C/5e04e969a5e84e1da2019df4463e5336, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/C/53a0278513ee4759aa44605b9735eebb, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/C/5de3d97f873c4881b6a1898aff51ec7a, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/C/708501389e9047549b8fc4029c12ce34, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/C/89c4a913ef464966be462584a775156d, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/C/2cbcf9a84c3d40f78e6f8c2ccce6ef4f, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/C/21f8ac0b64214560bb2e0b208542f540, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/C/134baf646ea8406ca000d85dc5aa31bf, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/C/f4e3a661402e40ea8860b469da3b9331, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/C/ac1d22e427fe4f85bc8b10a2d854a907, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/C/13c70f3b2e1f44b2bfa8a296f5b6e0f6, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/C/5c18e55bdda740b793020357fca9c24a, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/C/2f3b765abef44a36858f58b38e1db70d, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/C/fb5c30bfa8884523962e7df926400124] to archive 2024-11-23T16:15:08,664 DEBUG [StoreCloser-TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-23T16:15:08,666 DEBUG [StoreCloser-TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/C/011c9d7b98614d99b81fab8639fdf62d to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/C/011c9d7b98614d99b81fab8639fdf62d 2024-11-23T16:15:08,667 DEBUG [StoreCloser-TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/C/22c41e52183c4f37949452f6e1234d84 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/C/22c41e52183c4f37949452f6e1234d84 2024-11-23T16:15:08,668 DEBUG [StoreCloser-TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/C/cb95b6a55dd14f97986641f953711dc6 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/C/cb95b6a55dd14f97986641f953711dc6 2024-11-23T16:15:08,669 DEBUG [StoreCloser-TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/C/e131724238234ebca997fd18293eabfe to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/C/e131724238234ebca997fd18293eabfe 2024-11-23T16:15:08,670 DEBUG [StoreCloser-TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/C/0ec7a896d69c4ca1a2ab2571709796b6 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/C/0ec7a896d69c4ca1a2ab2571709796b6 2024-11-23T16:15:08,671 DEBUG [StoreCloser-TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/C/6aa31c68f85f450891c718706a92e59c to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/C/6aa31c68f85f450891c718706a92e59c 2024-11-23T16:15:08,672 DEBUG [StoreCloser-TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/C/367246ca68e04b1fb74befd857be030e to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/C/367246ca68e04b1fb74befd857be030e 2024-11-23T16:15:08,673 DEBUG [StoreCloser-TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/C/ff1f7c573de546d893d1db073b59e412 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/C/ff1f7c573de546d893d1db073b59e412 2024-11-23T16:15:08,674 DEBUG [StoreCloser-TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/C/a6bb103255e34ed98332935e32943a73 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/C/a6bb103255e34ed98332935e32943a73 2024-11-23T16:15:08,675 DEBUG [StoreCloser-TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/C/5e04e969a5e84e1da2019df4463e5336 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/C/5e04e969a5e84e1da2019df4463e5336 2024-11-23T16:15:08,676 DEBUG [StoreCloser-TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/C/53a0278513ee4759aa44605b9735eebb to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/C/53a0278513ee4759aa44605b9735eebb 2024-11-23T16:15:08,677 DEBUG [StoreCloser-TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/C/5de3d97f873c4881b6a1898aff51ec7a to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/C/5de3d97f873c4881b6a1898aff51ec7a 2024-11-23T16:15:08,678 DEBUG [StoreCloser-TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/C/708501389e9047549b8fc4029c12ce34 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/C/708501389e9047549b8fc4029c12ce34 2024-11-23T16:15:08,678 DEBUG [StoreCloser-TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/C/89c4a913ef464966be462584a775156d to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/C/89c4a913ef464966be462584a775156d 2024-11-23T16:15:08,679 DEBUG [StoreCloser-TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/C/2cbcf9a84c3d40f78e6f8c2ccce6ef4f to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/C/2cbcf9a84c3d40f78e6f8c2ccce6ef4f 2024-11-23T16:15:08,680 DEBUG [StoreCloser-TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/C/21f8ac0b64214560bb2e0b208542f540 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/C/21f8ac0b64214560bb2e0b208542f540 2024-11-23T16:15:08,682 DEBUG [StoreCloser-TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/C/134baf646ea8406ca000d85dc5aa31bf to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/C/134baf646ea8406ca000d85dc5aa31bf 2024-11-23T16:15:08,683 DEBUG [StoreCloser-TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/C/f4e3a661402e40ea8860b469da3b9331 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/C/f4e3a661402e40ea8860b469da3b9331 2024-11-23T16:15:08,684 DEBUG [StoreCloser-TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/C/ac1d22e427fe4f85bc8b10a2d854a907 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/C/ac1d22e427fe4f85bc8b10a2d854a907 2024-11-23T16:15:08,685 DEBUG [StoreCloser-TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/C/13c70f3b2e1f44b2bfa8a296f5b6e0f6 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/C/13c70f3b2e1f44b2bfa8a296f5b6e0f6 2024-11-23T16:15:08,686 DEBUG [StoreCloser-TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/C/5c18e55bdda740b793020357fca9c24a to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/C/5c18e55bdda740b793020357fca9c24a 2024-11-23T16:15:08,687 DEBUG [StoreCloser-TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/C/2f3b765abef44a36858f58b38e1db70d to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/C/2f3b765abef44a36858f58b38e1db70d 2024-11-23T16:15:08,688 DEBUG [StoreCloser-TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/C/fb5c30bfa8884523962e7df926400124 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/C/fb5c30bfa8884523962e7df926400124 2024-11-23T16:15:08,692 DEBUG [RS_CLOSE_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/recovered.edits/369.seqid, newMaxSeqId=369, maxSeqId=1 2024-11-23T16:15:08,692 INFO [RS_CLOSE_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e. 2024-11-23T16:15:08,693 DEBUG [RS_CLOSE_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.HRegion(1676): Region close journal for 5b34e731541fe76c476f23c5dc5aa28e: Waiting for close lock at 1732378507370Running coprocessor pre-close hooks at 1732378507370Disabling compacts and flushes for region at 1732378507370Disabling writes for close at 1732378507371 (+1 ms)Obtaining lock to block concurrent updates at 1732378507371Preparing flush snapshotting stores in 5b34e731541fe76c476f23c5dc5aa28e at 1732378507371Finished memstore snapshotting TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1732378507371Flushing stores of TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e. at 1732378507372 (+1 ms)Flushing 5b34e731541fe76c476f23c5dc5aa28e/A: creating writer at 1732378507372Flushing 5b34e731541fe76c476f23c5dc5aa28e/A: appending metadata at 1732378507374 (+2 ms)Flushing 5b34e731541fe76c476f23c5dc5aa28e/A: closing flushed file at 1732378507374Flushing 5b34e731541fe76c476f23c5dc5aa28e/B: creating writer at 1732378507783 (+409 ms)Flushing 5b34e731541fe76c476f23c5dc5aa28e/B: appending metadata at 1732378507785 (+2 ms)Flushing 5b34e731541fe76c476f23c5dc5aa28e/B: closing flushed file at 1732378507785Flushing 5b34e731541fe76c476f23c5dc5aa28e/C: creating writer at 1732378508193 (+408 ms)Flushing 5b34e731541fe76c476f23c5dc5aa28e/C: appending metadata at 1732378508195 (+2 ms)Flushing 5b34e731541fe76c476f23c5dc5aa28e/C: closing flushed file at 1732378508195Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1740b848: reopening flushed file at 1732378508603 (+408 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@532d4dd: reopening flushed file at 1732378508607 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@46c8f2d1: reopening flushed file at 1732378508611 (+4 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 5b34e731541fe76c476f23c5dc5aa28e in 1245ms, sequenceid=366, compaction requested=true at 1732378508616 (+5 ms)Writing region close event to WAL at 1732378508689 (+73 ms)Running coprocessor post-close hooks at 1732378508692 (+3 ms)Closed at 1732378508692 2024-11-23T16:15:08,695 INFO [RS_CLOSE_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] handler.UnassignRegionHandler(157): Closed 5b34e731541fe76c476f23c5dc5aa28e 2024-11-23T16:15:08,695 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=78 updating hbase:meta row=5b34e731541fe76c476f23c5dc5aa28e, regionState=CLOSED 2024-11-23T16:15:08,696 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=79, ppid=78, state=RUNNABLE, hasLock=false; CloseRegionProcedure 5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 because future has completed 2024-11-23T16:15:08,698 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=79, resume processing ppid=78 2024-11-23T16:15:08,698 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=79, ppid=78, state=SUCCESS, hasLock=false; CloseRegionProcedure 5b34e731541fe76c476f23c5dc5aa28e, server=1a7c5470b472,44715,1732378420375 in 1.4790 sec 2024-11-23T16:15:08,700 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=78, resume processing ppid=77 2024-11-23T16:15:08,700 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=78, ppid=77, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5b34e731541fe76c476f23c5dc5aa28e, UNASSIGN in 1.4820 sec 2024-11-23T16:15:08,702 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=77, resume processing ppid=76 2024-11-23T16:15:08,702 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=77, ppid=76, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.4850 sec 2024-11-23T16:15:08,703 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732378508703"}]},"ts":"1732378508703"} 2024-11-23T16:15:08,704 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-23T16:15:08,704 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-23T16:15:08,705 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=76, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.4950 sec 2024-11-23T16:15:09,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-23T16:15:09,359 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-23T16:15:09,360 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-23T16:15:09,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] procedure2.ProcedureExecutor(1139): Stored pid=80, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-23T16:15:09,362 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=80, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-23T16:15:09,363 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=80, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-23T16:15:09,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-23T16:15:09,369 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e 2024-11-23T16:15:09,375 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/A, FileablePath, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/B, FileablePath, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/C, FileablePath, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/recovered.edits] 2024-11-23T16:15:09,382 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/A/b16d64b54fb24938aba13b0ae2594b04 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/A/b16d64b54fb24938aba13b0ae2594b04 2024-11-23T16:15:09,385 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/A/d23a200809414ae9ab2c443e5d86aa47 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/A/d23a200809414ae9ab2c443e5d86aa47 2024-11-23T16:15:09,387 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/A/f02dece50b324b9dabd33db13263eb5a to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/A/f02dece50b324b9dabd33db13263eb5a 2024-11-23T16:15:09,394 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/B/0b07f17c24394a589819a6bc19ae5a34 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/B/0b07f17c24394a589819a6bc19ae5a34 2024-11-23T16:15:09,396 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/B/67042741dff945388c2a726ac6428f4e to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/B/67042741dff945388c2a726ac6428f4e 2024-11-23T16:15:09,397 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/B/81da43f44a7e4d44b7caddf8bf28dd9d to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/B/81da43f44a7e4d44b7caddf8bf28dd9d 2024-11-23T16:15:09,401 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/C/7fac4a050a104da1ad2136ef7dfdb5e7 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/C/7fac4a050a104da1ad2136ef7dfdb5e7 2024-11-23T16:15:09,402 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/C/8ffabef8bade4d6a9c366283fda636c4 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/C/8ffabef8bade4d6a9c366283fda636c4 2024-11-23T16:15:09,403 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/C/df34411f41b544b5a8323f37cadbf52e to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/C/df34411f41b544b5a8323f37cadbf52e 2024-11-23T16:15:09,406 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/recovered.edits/369.seqid to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e/recovered.edits/369.seqid 2024-11-23T16:15:09,407 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/5b34e731541fe76c476f23c5dc5aa28e 2024-11-23T16:15:09,407 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-23T16:15:09,410 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=80, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-23T16:15:09,416 WARN [PEWorker-4 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-23T16:15:09,420 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-23T16:15:09,422 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=80, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-23T16:15:09,422 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-23T16:15:09,422 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732378509422"}]},"ts":"9223372036854775807"} 2024-11-23T16:15:09,425 INFO [PEWorker-4 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-23T16:15:09,425 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 5b34e731541fe76c476f23c5dc5aa28e, NAME => 'TestAcidGuarantees,,1732378479338.5b34e731541fe76c476f23c5dc5aa28e.', STARTKEY => '', ENDKEY => ''}] 2024-11-23T16:15:09,425 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-23T16:15:09,425 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732378509425"}]},"ts":"9223372036854775807"} 2024-11-23T16:15:09,428 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-23T16:15:09,429 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(133): Finished pid=80, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-23T16:15:09,430 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=80, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 69 msec 2024-11-23T16:15:09,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-23T16:15:09,468 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-23T16:15:09,468 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-23T16:15:09,479 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testGetAtomicity Thread=248 (was 249), OpenFileDescriptor=451 (was 457), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=404 (was 435), ProcessCount=11 (was 11), AvailableMemoryMB=7306 (was 8069) 2024-11-23T16:15:09,489 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobScanAtomicity Thread=248, OpenFileDescriptor=451, MaxFileDescriptor=1048576, SystemLoadAverage=404, ProcessCount=11, AvailableMemoryMB=7305 2024-11-23T16:15:09,490 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-23T16:15:09,491 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-23T16:15:09,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] procedure2.ProcedureExecutor(1139): Stored pid=81, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-23T16:15:09,493 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=81, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-23T16:15:09,493 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:09,493 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 81 2024-11-23T16:15:09,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=81 2024-11-23T16:15:09,494 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=81, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-23T16:15:09,499 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742140_1316 (size=1061) 2024-11-23T16:15:09,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=81 2024-11-23T16:15:09,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=81 2024-11-23T16:15:09,901 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 0b4ea01214996120537d91f0bab5da66, NAME => 'TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca 2024-11-23T16:15:09,905 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742141_1317 (size=53) 2024-11-23T16:15:10,108 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-23T16:15:10,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=81 2024-11-23T16:15:10,306 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T16:15:10,307 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 0b4ea01214996120537d91f0bab5da66, disabling compactions & flushes 2024-11-23T16:15:10,307 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66. 2024-11-23T16:15:10,307 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66. 2024-11-23T16:15:10,307 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66. after waiting 0 ms 2024-11-23T16:15:10,307 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66. 2024-11-23T16:15:10,307 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66. 2024-11-23T16:15:10,307 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 0b4ea01214996120537d91f0bab5da66: Waiting for close lock at 1732378510307Disabling compacts and flushes for region at 1732378510307Disabling writes for close at 1732378510307Writing region close event to WAL at 1732378510307Closed at 1732378510307 2024-11-23T16:15:10,308 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=81, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-23T16:15:10,308 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732378510308"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732378510308"}]},"ts":"1732378510308"} 2024-11-23T16:15:10,310 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-23T16:15:10,310 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=81, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-23T16:15:10,311 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732378510310"}]},"ts":"1732378510310"} 2024-11-23T16:15:10,312 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-23T16:15:10,312 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=82, ppid=81, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=0b4ea01214996120537d91f0bab5da66, ASSIGN}] 2024-11-23T16:15:10,313 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=82, ppid=81, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=0b4ea01214996120537d91f0bab5da66, ASSIGN 2024-11-23T16:15:10,313 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=82, ppid=81, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=0b4ea01214996120537d91f0bab5da66, ASSIGN; state=OFFLINE, location=1a7c5470b472,44715,1732378420375; forceNewPlan=false, retain=false 2024-11-23T16:15:10,464 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=82 updating hbase:meta row=0b4ea01214996120537d91f0bab5da66, regionState=OPENING, regionLocation=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:10,466 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=82, ppid=81, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=0b4ea01214996120537d91f0bab5da66, ASSIGN because future has completed 2024-11-23T16:15:10,466 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=83, ppid=82, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375}] 2024-11-23T16:15:10,620 INFO [RS_OPEN_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66. 2024-11-23T16:15:10,620 DEBUG [RS_OPEN_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.HRegion(7752): Opening region: {ENCODED => 0b4ea01214996120537d91f0bab5da66, NAME => 'TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66.', STARTKEY => '', ENDKEY => ''} 2024-11-23T16:15:10,621 DEBUG [RS_OPEN_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 0b4ea01214996120537d91f0bab5da66 2024-11-23T16:15:10,621 DEBUG [RS_OPEN_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T16:15:10,621 DEBUG [RS_OPEN_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.HRegion(7794): checking encryption for 0b4ea01214996120537d91f0bab5da66 2024-11-23T16:15:10,621 DEBUG [RS_OPEN_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.HRegion(7797): checking classloading for 0b4ea01214996120537d91f0bab5da66 2024-11-23T16:15:10,622 INFO [StoreOpener-0b4ea01214996120537d91f0bab5da66-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 0b4ea01214996120537d91f0bab5da66 2024-11-23T16:15:10,623 INFO [StoreOpener-0b4ea01214996120537d91f0bab5da66-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-23T16:15:10,623 INFO [StoreOpener-0b4ea01214996120537d91f0bab5da66-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0b4ea01214996120537d91f0bab5da66 columnFamilyName A 2024-11-23T16:15:10,623 DEBUG [StoreOpener-0b4ea01214996120537d91f0bab5da66-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:10,624 INFO [StoreOpener-0b4ea01214996120537d91f0bab5da66-1 {}] regionserver.HStore(327): Store=0b4ea01214996120537d91f0bab5da66/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T16:15:10,624 INFO [StoreOpener-0b4ea01214996120537d91f0bab5da66-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 0b4ea01214996120537d91f0bab5da66 2024-11-23T16:15:10,625 INFO [StoreOpener-0b4ea01214996120537d91f0bab5da66-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-23T16:15:10,625 INFO [StoreOpener-0b4ea01214996120537d91f0bab5da66-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0b4ea01214996120537d91f0bab5da66 columnFamilyName B 2024-11-23T16:15:10,625 DEBUG [StoreOpener-0b4ea01214996120537d91f0bab5da66-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:10,625 INFO [StoreOpener-0b4ea01214996120537d91f0bab5da66-1 {}] regionserver.HStore(327): Store=0b4ea01214996120537d91f0bab5da66/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T16:15:10,625 INFO [StoreOpener-0b4ea01214996120537d91f0bab5da66-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 0b4ea01214996120537d91f0bab5da66 2024-11-23T16:15:10,626 INFO [StoreOpener-0b4ea01214996120537d91f0bab5da66-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-23T16:15:10,626 INFO [StoreOpener-0b4ea01214996120537d91f0bab5da66-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0b4ea01214996120537d91f0bab5da66 columnFamilyName C 2024-11-23T16:15:10,626 DEBUG [StoreOpener-0b4ea01214996120537d91f0bab5da66-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:10,627 INFO [StoreOpener-0b4ea01214996120537d91f0bab5da66-1 {}] regionserver.HStore(327): Store=0b4ea01214996120537d91f0bab5da66/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T16:15:10,627 INFO [RS_OPEN_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66. 2024-11-23T16:15:10,627 DEBUG [RS_OPEN_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.HRegion(1038): replaying wal for 0b4ea01214996120537d91f0bab5da66 2024-11-23T16:15:10,627 DEBUG [RS_OPEN_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66 2024-11-23T16:15:10,628 DEBUG [RS_OPEN_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66 2024-11-23T16:15:10,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=81 2024-11-23T16:15:10,629 DEBUG [RS_OPEN_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.HRegion(1048): stopping wal replay for 0b4ea01214996120537d91f0bab5da66 2024-11-23T16:15:10,629 DEBUG [RS_OPEN_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.HRegion(1060): Cleaning up temporary data for 0b4ea01214996120537d91f0bab5da66 2024-11-23T16:15:10,629 DEBUG [RS_OPEN_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-23T16:15:10,630 DEBUG [RS_OPEN_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.HRegion(1093): writing seq id for 0b4ea01214996120537d91f0bab5da66 2024-11-23T16:15:10,632 DEBUG [RS_OPEN_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-23T16:15:10,632 INFO [RS_OPEN_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.HRegion(1114): Opened 0b4ea01214996120537d91f0bab5da66; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66548341, jitterRate=-0.00835244357585907}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-23T16:15:10,632 DEBUG [RS_OPEN_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 0b4ea01214996120537d91f0bab5da66 2024-11-23T16:15:10,633 DEBUG [RS_OPEN_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.HRegion(1006): Region open journal for 0b4ea01214996120537d91f0bab5da66: Running coprocessor pre-open hook at 1732378510621Writing region info on filesystem at 1732378510621Initializing all the Stores at 1732378510622 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732378510622Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732378510622Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732378510622Cleaning up temporary data from old regions at 1732378510629 (+7 ms)Running coprocessor post-open hooks at 1732378510632 (+3 ms)Region opened successfully at 1732378510633 (+1 ms) 2024-11-23T16:15:10,633 INFO [RS_OPEN_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., pid=83, masterSystemTime=1732378510617 2024-11-23T16:15:10,635 DEBUG [RS_OPEN_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66. 2024-11-23T16:15:10,635 INFO [RS_OPEN_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66. 2024-11-23T16:15:10,635 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=82 updating hbase:meta row=0b4ea01214996120537d91f0bab5da66, regionState=OPEN, openSeqNum=2, regionLocation=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:10,636 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=83, ppid=82, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 because future has completed 2024-11-23T16:15:10,637 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45799 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=1a7c5470b472,44715,1732378420375, table=TestAcidGuarantees, region=0b4ea01214996120537d91f0bab5da66. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-23T16:15:10,638 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=83, resume processing ppid=82 2024-11-23T16:15:10,638 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=83, ppid=82, state=SUCCESS, hasLock=false; OpenRegionProcedure 0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 in 171 msec 2024-11-23T16:15:10,640 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=82, resume processing ppid=81 2024-11-23T16:15:10,640 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=82, ppid=81, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=0b4ea01214996120537d91f0bab5da66, ASSIGN in 326 msec 2024-11-23T16:15:10,641 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=81, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-23T16:15:10,641 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732378510641"}]},"ts":"1732378510641"} 2024-11-23T16:15:10,642 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-23T16:15:10,642 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=81, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-23T16:15:10,644 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=81, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1520 sec 2024-11-23T16:15:11,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=81 2024-11-23T16:15:11,638 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-23T16:15:11,639 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1af6f666, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T16:15:11,640 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1a7c5470b472,45799,-1 for getting cluster id 2024-11-23T16:15:11,640 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-23T16:15:11,641 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '34e0592b-548a-4d73-be6a-df1bafadfd92' 2024-11-23T16:15:11,641 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-23T16:15:11,641 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "34e0592b-548a-4d73-be6a-df1bafadfd92" 2024-11-23T16:15:11,641 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@77d8bb45, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T16:15:11,641 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1a7c5470b472,45799,-1] 2024-11-23T16:15:11,641 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-23T16:15:11,642 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T16:15:11,642 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46108, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-23T16:15:11,643 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4ba9107b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T16:15:11,643 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T16:15:11,644 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1a7c5470b472,44715,1732378420375, seqNum=-1] 2024-11-23T16:15:11,644 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T16:15:11,645 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55392, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T16:15:11,646 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-23T16:15:11,646 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 1a7c5470b472,45799,1732378419617 2024-11-23T16:15:11,646 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@2e4eeaf8 2024-11-23T16:15:11,646 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-23T16:15:11,647 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46124, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-23T16:15:11,648 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-23T16:15:11,649 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-23T16:15:11,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] procedure2.ProcedureExecutor(1139): Stored pid=84, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-23T16:15:11,659 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742142_1318 (size=1097) 2024-11-23T16:15:12,061 DEBUG [PEWorker-5 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-11-23T16:15:12,061 INFO [PEWorker-5 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-11-23T16:15:12,063 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=85, ppid=84, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-23T16:15:12,065 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=86, ppid=85, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=0b4ea01214996120537d91f0bab5da66, REOPEN/MOVE}] 2024-11-23T16:15:12,066 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=86, ppid=85, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=0b4ea01214996120537d91f0bab5da66, REOPEN/MOVE 2024-11-23T16:15:12,066 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=86 updating hbase:meta row=0b4ea01214996120537d91f0bab5da66, regionState=CLOSING, regionLocation=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:12,067 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=86, ppid=85, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=0b4ea01214996120537d91f0bab5da66, REOPEN/MOVE because future has completed 2024-11-23T16:15:12,067 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-23T16:15:12,067 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=87, ppid=86, state=RUNNABLE, hasLock=false; CloseRegionProcedure 0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375}] 2024-11-23T16:15:12,219 INFO [RS_CLOSE_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] handler.UnassignRegionHandler(122): Close 0b4ea01214996120537d91f0bab5da66 2024-11-23T16:15:12,220 DEBUG [RS_CLOSE_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-23T16:15:12,220 DEBUG [RS_CLOSE_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(1722): Closing 0b4ea01214996120537d91f0bab5da66, disabling compactions & flushes 2024-11-23T16:15:12,220 INFO [RS_CLOSE_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66. 2024-11-23T16:15:12,220 DEBUG [RS_CLOSE_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66. 2024-11-23T16:15:12,220 DEBUG [RS_CLOSE_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66. after waiting 0 ms 2024-11-23T16:15:12,220 DEBUG [RS_CLOSE_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66. 2024-11-23T16:15:12,223 DEBUG [RS_CLOSE_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-23T16:15:12,224 INFO [RS_CLOSE_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66. 2024-11-23T16:15:12,224 DEBUG [RS_CLOSE_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(1676): Region close journal for 0b4ea01214996120537d91f0bab5da66: Waiting for close lock at 1732378512220Running coprocessor pre-close hooks at 1732378512220Disabling compacts and flushes for region at 1732378512220Disabling writes for close at 1732378512220Writing region close event to WAL at 1732378512220Running coprocessor post-close hooks at 1732378512224 (+4 ms)Closed at 1732378512224 2024-11-23T16:15:12,224 WARN [RS_CLOSE_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegionServer(3299): Not adding moved region record: 0b4ea01214996120537d91f0bab5da66 to self. 2024-11-23T16:15:12,225 INFO [RS_CLOSE_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] handler.UnassignRegionHandler(157): Closed 0b4ea01214996120537d91f0bab5da66 2024-11-23T16:15:12,226 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=86 updating hbase:meta row=0b4ea01214996120537d91f0bab5da66, regionState=CLOSED 2024-11-23T16:15:12,227 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=87, ppid=86, state=RUNNABLE, hasLock=false; CloseRegionProcedure 0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 because future has completed 2024-11-23T16:15:12,229 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=87, resume processing ppid=86 2024-11-23T16:15:12,229 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=87, ppid=86, state=SUCCESS, hasLock=false; CloseRegionProcedure 0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 in 160 msec 2024-11-23T16:15:12,229 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=86, ppid=85, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=0b4ea01214996120537d91f0bab5da66, REOPEN/MOVE; state=CLOSED, location=1a7c5470b472,44715,1732378420375; forceNewPlan=false, retain=true 2024-11-23T16:15:12,380 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=86 updating hbase:meta row=0b4ea01214996120537d91f0bab5da66, regionState=OPENING, regionLocation=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:12,381 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=86, ppid=85, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=0b4ea01214996120537d91f0bab5da66, REOPEN/MOVE because future has completed 2024-11-23T16:15:12,382 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=88, ppid=86, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375}] 2024-11-23T16:15:12,536 INFO [RS_OPEN_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66. 2024-11-23T16:15:12,536 DEBUG [RS_OPEN_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(7752): Opening region: {ENCODED => 0b4ea01214996120537d91f0bab5da66, NAME => 'TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66.', STARTKEY => '', ENDKEY => ''} 2024-11-23T16:15:12,536 DEBUG [RS_OPEN_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 0b4ea01214996120537d91f0bab5da66 2024-11-23T16:15:12,536 DEBUG [RS_OPEN_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T16:15:12,536 DEBUG [RS_OPEN_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(7794): checking encryption for 0b4ea01214996120537d91f0bab5da66 2024-11-23T16:15:12,536 DEBUG [RS_OPEN_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(7797): checking classloading for 0b4ea01214996120537d91f0bab5da66 2024-11-23T16:15:12,538 INFO [StoreOpener-0b4ea01214996120537d91f0bab5da66-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 0b4ea01214996120537d91f0bab5da66 2024-11-23T16:15:12,538 INFO [StoreOpener-0b4ea01214996120537d91f0bab5da66-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-23T16:15:12,538 INFO [StoreOpener-0b4ea01214996120537d91f0bab5da66-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0b4ea01214996120537d91f0bab5da66 columnFamilyName A 2024-11-23T16:15:12,539 DEBUG [StoreOpener-0b4ea01214996120537d91f0bab5da66-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:12,540 INFO [StoreOpener-0b4ea01214996120537d91f0bab5da66-1 {}] regionserver.HStore(327): Store=0b4ea01214996120537d91f0bab5da66/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T16:15:12,540 INFO [StoreOpener-0b4ea01214996120537d91f0bab5da66-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 0b4ea01214996120537d91f0bab5da66 2024-11-23T16:15:12,541 INFO [StoreOpener-0b4ea01214996120537d91f0bab5da66-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-23T16:15:12,541 INFO [StoreOpener-0b4ea01214996120537d91f0bab5da66-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0b4ea01214996120537d91f0bab5da66 columnFamilyName B 2024-11-23T16:15:12,541 DEBUG [StoreOpener-0b4ea01214996120537d91f0bab5da66-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:12,541 INFO [StoreOpener-0b4ea01214996120537d91f0bab5da66-1 {}] regionserver.HStore(327): Store=0b4ea01214996120537d91f0bab5da66/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T16:15:12,541 INFO [StoreOpener-0b4ea01214996120537d91f0bab5da66-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 0b4ea01214996120537d91f0bab5da66 2024-11-23T16:15:12,541 INFO [StoreOpener-0b4ea01214996120537d91f0bab5da66-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-23T16:15:12,542 INFO [StoreOpener-0b4ea01214996120537d91f0bab5da66-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0b4ea01214996120537d91f0bab5da66 columnFamilyName C 2024-11-23T16:15:12,542 DEBUG [StoreOpener-0b4ea01214996120537d91f0bab5da66-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:12,542 INFO [StoreOpener-0b4ea01214996120537d91f0bab5da66-1 {}] regionserver.HStore(327): Store=0b4ea01214996120537d91f0bab5da66/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T16:15:12,542 INFO [RS_OPEN_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66. 2024-11-23T16:15:12,542 DEBUG [RS_OPEN_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(1038): replaying wal for 0b4ea01214996120537d91f0bab5da66 2024-11-23T16:15:12,543 DEBUG [RS_OPEN_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66 2024-11-23T16:15:12,543 DEBUG [RS_OPEN_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66 2024-11-23T16:15:12,544 DEBUG [RS_OPEN_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(1048): stopping wal replay for 0b4ea01214996120537d91f0bab5da66 2024-11-23T16:15:12,544 DEBUG [RS_OPEN_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(1060): Cleaning up temporary data for 0b4ea01214996120537d91f0bab5da66 2024-11-23T16:15:12,544 DEBUG [RS_OPEN_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-23T16:15:12,545 DEBUG [RS_OPEN_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(1093): writing seq id for 0b4ea01214996120537d91f0bab5da66 2024-11-23T16:15:12,546 INFO [RS_OPEN_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(1114): Opened 0b4ea01214996120537d91f0bab5da66; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63785443, jitterRate=-0.04952283203601837}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-23T16:15:12,546 DEBUG [RS_OPEN_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 0b4ea01214996120537d91f0bab5da66 2024-11-23T16:15:12,547 DEBUG [RS_OPEN_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(1006): Region open journal for 0b4ea01214996120537d91f0bab5da66: Running coprocessor pre-open hook at 1732378512537Writing region info on filesystem at 1732378512537Initializing all the Stores at 1732378512537Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732378512537Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732378512537Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732378512537Cleaning up temporary data from old regions at 1732378512544 (+7 ms)Running coprocessor post-open hooks at 1732378512546 (+2 ms)Region opened successfully at 1732378512547 (+1 ms) 2024-11-23T16:15:12,547 INFO [RS_OPEN_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., pid=88, masterSystemTime=1732378512533 2024-11-23T16:15:12,549 DEBUG [RS_OPEN_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66. 2024-11-23T16:15:12,549 INFO [RS_OPEN_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66. 2024-11-23T16:15:12,549 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=86 updating hbase:meta row=0b4ea01214996120537d91f0bab5da66, regionState=OPEN, openSeqNum=5, regionLocation=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:12,551 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=88, ppid=86, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 because future has completed 2024-11-23T16:15:12,553 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=88, resume processing ppid=86 2024-11-23T16:15:12,553 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=88, ppid=86, state=SUCCESS, hasLock=false; OpenRegionProcedure 0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 in 170 msec 2024-11-23T16:15:12,554 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=86, resume processing ppid=85 2024-11-23T16:15:12,554 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=86, ppid=85, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=0b4ea01214996120537d91f0bab5da66, REOPEN/MOVE in 488 msec 2024-11-23T16:15:12,556 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=85, resume processing ppid=84 2024-11-23T16:15:12,556 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=85, ppid=84, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 491 msec 2024-11-23T16:15:12,558 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=84, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 907 msec 2024-11-23T16:15:12,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-23T16:15:12,559 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-23T16:15:12,559 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@9ff38b5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T16:15:12,559 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1a7c5470b472,45799,-1 for getting cluster id 2024-11-23T16:15:12,559 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-23T16:15:12,560 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '34e0592b-548a-4d73-be6a-df1bafadfd92' 2024-11-23T16:15:12,560 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-23T16:15:12,560 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "34e0592b-548a-4d73-be6a-df1bafadfd92" 2024-11-23T16:15:12,560 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6081f862, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T16:15:12,560 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1a7c5470b472,45799,-1] 2024-11-23T16:15:12,561 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-23T16:15:12,561 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T16:15:12,561 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46144, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-23T16:15:12,562 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@760f6f85, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T16:15:12,562 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@62ec6412, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T16:15:12,562 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1a7c5470b472,45799,-1 for getting cluster id 2024-11-23T16:15:12,563 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-23T16:15:12,563 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '34e0592b-548a-4d73-be6a-df1bafadfd92' 2024-11-23T16:15:12,563 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-23T16:15:12,563 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "34e0592b-548a-4d73-be6a-df1bafadfd92" 2024-11-23T16:15:12,563 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@606d4aed, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T16:15:12,563 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1a7c5470b472,45799,-1] 2024-11-23T16:15:12,564 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-23T16:15:12,564 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T16:15:12,564 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46158, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-23T16:15:12,565 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5c8fff57, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T16:15:12,565 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@37f2378c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T16:15:12,565 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1a7c5470b472,45799,-1 for getting cluster id 2024-11-23T16:15:12,565 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-23T16:15:12,566 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '34e0592b-548a-4d73-be6a-df1bafadfd92' 2024-11-23T16:15:12,566 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-23T16:15:12,566 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "34e0592b-548a-4d73-be6a-df1bafadfd92" 2024-11-23T16:15:12,566 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2573871f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T16:15:12,566 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1a7c5470b472,45799,-1] 2024-11-23T16:15:12,566 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-23T16:15:12,566 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T16:15:12,567 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46174, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-23T16:15:12,567 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f9a06b8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T16:15:12,568 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@43cc49f7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T16:15:12,568 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1a7c5470b472,45799,-1 for getting cluster id 2024-11-23T16:15:12,568 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-23T16:15:12,568 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '34e0592b-548a-4d73-be6a-df1bafadfd92' 2024-11-23T16:15:12,569 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-23T16:15:12,569 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "34e0592b-548a-4d73-be6a-df1bafadfd92" 2024-11-23T16:15:12,569 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@791b5114, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T16:15:12,569 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1a7c5470b472,45799,-1] 2024-11-23T16:15:12,569 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-23T16:15:12,569 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T16:15:12,569 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46204, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-23T16:15:12,570 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f29032a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T16:15:12,570 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e0f0eef, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T16:15:12,570 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1a7c5470b472,45799,-1 for getting cluster id 2024-11-23T16:15:12,570 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-23T16:15:12,571 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '34e0592b-548a-4d73-be6a-df1bafadfd92' 2024-11-23T16:15:12,571 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-23T16:15:12,571 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "34e0592b-548a-4d73-be6a-df1bafadfd92" 2024-11-23T16:15:12,571 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3699b0b2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T16:15:12,571 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1a7c5470b472,45799,-1] 2024-11-23T16:15:12,571 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-23T16:15:12,571 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T16:15:12,572 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46224, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-23T16:15:12,572 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@36680863, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T16:15:12,573 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@65616af9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T16:15:12,573 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1a7c5470b472,45799,-1 for getting cluster id 2024-11-23T16:15:12,573 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-23T16:15:12,573 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '34e0592b-548a-4d73-be6a-df1bafadfd92' 2024-11-23T16:15:12,574 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-23T16:15:12,574 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "34e0592b-548a-4d73-be6a-df1bafadfd92" 2024-11-23T16:15:12,574 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@bb17693, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T16:15:12,574 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1a7c5470b472,45799,-1] 2024-11-23T16:15:12,574 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-23T16:15:12,574 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T16:15:12,574 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46252, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-23T16:15:12,575 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@23b126e5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T16:15:12,575 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@74481d40, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T16:15:12,575 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1a7c5470b472,45799,-1 for getting cluster id 2024-11-23T16:15:12,575 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-23T16:15:12,576 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '34e0592b-548a-4d73-be6a-df1bafadfd92' 2024-11-23T16:15:12,576 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-23T16:15:12,576 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "34e0592b-548a-4d73-be6a-df1bafadfd92" 2024-11-23T16:15:12,576 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@65bd4ddb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T16:15:12,576 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1a7c5470b472,45799,-1] 2024-11-23T16:15:12,576 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-23T16:15:12,576 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T16:15:12,577 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46272, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-23T16:15:12,577 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@23aa0c56, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T16:15:12,578 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@514b3d51, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T16:15:12,578 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1a7c5470b472,45799,-1 for getting cluster id 2024-11-23T16:15:12,578 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-23T16:15:12,578 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '34e0592b-548a-4d73-be6a-df1bafadfd92' 2024-11-23T16:15:12,578 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-23T16:15:12,578 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "34e0592b-548a-4d73-be6a-df1bafadfd92" 2024-11-23T16:15:12,579 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3ec359e1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T16:15:12,579 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1a7c5470b472,45799,-1] 2024-11-23T16:15:12,579 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-23T16:15:12,579 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T16:15:12,579 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46292, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-23T16:15:12,580 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1f679a8d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T16:15:12,580 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1b14e3fc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T16:15:12,580 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1a7c5470b472,45799,-1 for getting cluster id 2024-11-23T16:15:12,580 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-23T16:15:12,581 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '34e0592b-548a-4d73-be6a-df1bafadfd92' 2024-11-23T16:15:12,581 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-23T16:15:12,581 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "34e0592b-548a-4d73-be6a-df1bafadfd92" 2024-11-23T16:15:12,581 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1b5bb9f5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T16:15:12,581 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1a7c5470b472,45799,-1] 2024-11-23T16:15:12,581 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-23T16:15:12,581 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T16:15:12,582 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46300, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-23T16:15:12,582 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3b90466e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T16:15:12,583 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@75e264c3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T16:15:12,583 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1a7c5470b472,45799,-1 for getting cluster id 2024-11-23T16:15:12,583 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-23T16:15:12,583 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '34e0592b-548a-4d73-be6a-df1bafadfd92' 2024-11-23T16:15:12,584 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-23T16:15:12,584 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "34e0592b-548a-4d73-be6a-df1bafadfd92" 2024-11-23T16:15:12,584 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@319716fa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T16:15:12,584 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1a7c5470b472,45799,-1] 2024-11-23T16:15:12,584 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-23T16:15:12,584 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T16:15:12,585 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46316, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-23T16:15:12,585 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c5b332a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T16:15:12,586 DEBUG [Thread-1460 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T16:15:12,586 DEBUG [Thread-1464 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T16:15:12,586 DEBUG [Thread-1479 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T16:15:12,586 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1a7c5470b472,44715,1732378420375, seqNum=-1] 2024-11-23T16:15:12,586 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T16:15:12,587 DEBUG [Thread-1473 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T16:15:12,587 DEBUG [Thread-1477 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T16:15:12,587 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1a7c5470b472,44715,1732378420375, seqNum=-1] 2024-11-23T16:15:12,587 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1a7c5470b472,44715,1732378420375, seqNum=-1] 2024-11-23T16:15:12,587 DEBUG [Thread-1468 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T16:15:12,587 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T16:15:12,587 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1a7c5470b472,44715,1732378420375, seqNum=-1] 2024-11-23T16:15:12,587 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T16:15:12,588 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T16:15:12,588 DEBUG [Thread-1471 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T16:15:12,588 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1a7c5470b472,44715,1732378420375, seqNum=-1] 2024-11-23T16:15:12,588 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45799 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-23T16:15:12,588 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T16:15:12,588 DEBUG [Thread-1475 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T16:15:12,588 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55404, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T16:15:12,589 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1a7c5470b472,44715,1732378420375, seqNum=-1] 2024-11-23T16:15:12,589 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T16:15:12,589 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55422, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T16:15:12,589 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55412, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T16:15:12,589 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55438, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T16:15:12,589 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1a7c5470b472,44715,1732378420375, seqNum=-1] 2024-11-23T16:15:12,589 DEBUG [Thread-1462 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T16:15:12,590 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T16:15:12,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45799 {}] procedure2.ProcedureExecutor(1139): Stored pid=89, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=89, table=TestAcidGuarantees 2024-11-23T16:15:12,590 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55450, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T16:15:12,590 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55454, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T16:15:12,590 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55462, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T16:15:12,590 DEBUG [Thread-1466 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T16:15:12,590 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1a7c5470b472,44715,1732378420375, seqNum=-1] 2024-11-23T16:15:12,591 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T16:15:12,591 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=89, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=89, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-23T16:15:12,591 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1a7c5470b472,44715,1732378420375, seqNum=-1] 2024-11-23T16:15:12,591 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55464, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T16:15:12,591 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=89, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=89, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-23T16:15:12,591 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T16:15:12,591 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=90, ppid=89, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-23T16:15:12,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-23T16:15:12,592 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5] 2024-11-23T16:15:12,593 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5] 2024-11-23T16:15:12,593 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55466, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T16:15:12,593 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5] 2024-11-23T16:15:12,595 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5] 2024-11-23T16:15:12,596 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5] 2024-11-23T16:15:12,597 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5] 2024-11-23T16:15:12,597 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5] 2024-11-23T16:15:12,598 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5] 2024-11-23T16:15:12,598 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1a7c5470b472,44715,1732378420375, seqNum=-1] 2024-11-23T16:15:12,599 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T16:15:12,599 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5] 2024-11-23T16:15:12,600 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55476, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T16:15:12,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(8855): Flush requested on 0b4ea01214996120537d91f0bab5da66 2024-11-23T16:15:12,601 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0b4ea01214996120537d91f0bab5da66 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-23T16:15:12,601 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0b4ea01214996120537d91f0bab5da66, store=A 2024-11-23T16:15:12,601 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5] 2024-11-23T16:15:12,601 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-23T16:15:12,601 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0b4ea01214996120537d91f0bab5da66, store=B 2024-11-23T16:15:12,601 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-23T16:15:12,601 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0b4ea01214996120537d91f0bab5da66, store=C 2024-11-23T16:15:12,601 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-23T16:15:12,638 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241123fd2eb836b9bb478aa74acb95866049c3_0b4ea01214996120537d91f0bab5da66 is 50, key is test_row_0/A:col10/1732378512593/Put/seqid=0 2024-11-23T16:15:12,649 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742143_1319 (size=12154) 2024-11-23T16:15:12,652 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1a7c5470b472:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732378512630, max timestamp=1732378512648Num uniques 859; 2024-11-23T16:15:12,652 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1a7c5470b472:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732378512630, max timestamp=1732378512648Num uniques 859; 2024-11-23T16:15:12,652 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1a7c5470b472:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732378512630, max timestamp=1732378512648Num uniques 859; 2024-11-23T16:15:12,652 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1a7c5470b472:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-23T16:15:12,652 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1a7c5470b472:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-23T16:15:12,652 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1a7c5470b472:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-23T16:15:12,692 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:12,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55422 deadline: 1732378572680, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:12,693 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:12,693 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:12,693 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:15:12,696 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:12,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55404 deadline: 1732378572681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:12,697 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:12,697 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:12,697 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:12,697 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:15:12,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55412 deadline: 1732378572682, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:12,697 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:12,697 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:12,697 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:12,697 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:15:12,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55466 deadline: 1732378572683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:12,698 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:12,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55464 deadline: 1732378572684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:12,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-23T16:15:12,698 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:12,698 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:12,698 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:15:12,699 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:12,699 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:12,700 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:15:12,744 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=90 2024-11-23T16:15:12,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66. 2024-11-23T16:15:12,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66. as already flushing 2024-11-23T16:15:12,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66. 2024-11-23T16:15:12,745 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] handler.RSProcedureHandler(58): pid=90 java.io.IOException: Unable to complete flush {ENCODED => 0b4ea01214996120537d91f0bab5da66, NAME => 'TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:15:12,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=90 java.io.IOException: Unable to complete flush {ENCODED => 0b4ea01214996120537d91f0bab5da66, NAME => 'TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:15:12,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.HMaster(4177): Remote procedure failed, pid=90 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0b4ea01214996120537d91f0bab5da66, NAME => 'TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0b4ea01214996120537d91f0bab5da66, NAME => 'TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:15:12,807 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:12,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55422 deadline: 1732378572798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:12,808 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:12,808 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:12,808 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:15:12,819 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:12,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55404 deadline: 1732378572808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:12,820 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:12,820 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:12,820 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:12,820 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:15:12,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55412 deadline: 1732378572809, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:12,820 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:12,820 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:12,820 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:12,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55466 deadline: 1732378572809, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:12,820 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:15:12,821 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:12,821 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:12,821 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:15:12,822 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:12,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55464 deadline: 1732378572809, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:12,822 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:12,822 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:12,822 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:15:12,897 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=90 2024-11-23T16:15:12,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66. 2024-11-23T16:15:12,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66. as already flushing 2024-11-23T16:15:12,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66. 2024-11-23T16:15:12,898 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] handler.RSProcedureHandler(58): pid=90 java.io.IOException: Unable to complete flush {ENCODED => 0b4ea01214996120537d91f0bab5da66, NAME => 'TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:15:12,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=90 java.io.IOException: Unable to complete flush {ENCODED => 0b4ea01214996120537d91f0bab5da66, NAME => 'TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:15:12,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.HMaster(4177): Remote procedure failed, pid=90 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0b4ea01214996120537d91f0bab5da66, NAME => 'TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0b4ea01214996120537d91f0bab5da66, NAME => 'TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:15:12,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-23T16:15:13,029 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:13,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55422 deadline: 1732378573018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:13,030 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:13,030 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:13,030 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:15:13,039 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:13,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55404 deadline: 1732378573028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:13,039 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:13,039 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:13,040 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:13,040 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:15:13,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55412 deadline: 1732378573028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:13,040 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:13,040 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:13,040 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:13,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55466 deadline: 1732378573028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:13,040 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:15:13,040 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:13,040 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:13,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55464 deadline: 1732378573028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:13,040 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:13,040 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:15:13,040 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:13,040 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:13,040 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:15:13,050 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:13,051 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=90 2024-11-23T16:15:13,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66. 2024-11-23T16:15:13,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66. as already flushing 2024-11-23T16:15:13,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66. 2024-11-23T16:15:13,051 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] handler.RSProcedureHandler(58): pid=90 java.io.IOException: Unable to complete flush {ENCODED => 0b4ea01214996120537d91f0bab5da66, NAME => 'TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:15:13,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=90 java.io.IOException: Unable to complete flush {ENCODED => 0b4ea01214996120537d91f0bab5da66, NAME => 'TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:15:13,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.HMaster(4177): Remote procedure failed, pid=90 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0b4ea01214996120537d91f0bab5da66, NAME => 'TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0b4ea01214996120537d91f0bab5da66, NAME => 'TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:15:13,054 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241123fd2eb836b9bb478aa74acb95866049c3_0b4ea01214996120537d91f0bab5da66 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123fd2eb836b9bb478aa74acb95866049c3_0b4ea01214996120537d91f0bab5da66 2024-11-23T16:15:13,055 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/.tmp/A/c16c26c373a14e1fb105eedc59b9a723, store: [table=TestAcidGuarantees family=A region=0b4ea01214996120537d91f0bab5da66] 2024-11-23T16:15:13,055 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/.tmp/A/c16c26c373a14e1fb105eedc59b9a723 is 175, key is test_row_0/A:col10/1732378512593/Put/seqid=0 2024-11-23T16:15:13,065 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742144_1320 (size=30955) 2024-11-23T16:15:13,204 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=90 2024-11-23T16:15:13,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66. 2024-11-23T16:15:13,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66. as already flushing 2024-11-23T16:15:13,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66. 2024-11-23T16:15:13,204 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] handler.RSProcedureHandler(58): pid=90 java.io.IOException: Unable to complete flush {ENCODED => 0b4ea01214996120537d91f0bab5da66, NAME => 'TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:15:13,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=90 java.io.IOException: Unable to complete flush {ENCODED => 0b4ea01214996120537d91f0bab5da66, NAME => 'TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:15:13,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.HMaster(4177): Remote procedure failed, pid=90 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0b4ea01214996120537d91f0bab5da66, NAME => 'TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0b4ea01214996120537d91f0bab5da66, NAME => 'TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:15:13,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-23T16:15:13,339 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:13,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55422 deadline: 1732378573338, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:13,340 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:13,340 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:13,340 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:15:13,355 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:13,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55404 deadline: 1732378573348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:13,355 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:13,355 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:13,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55412 deadline: 1732378573348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:13,355 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:13,355 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:15:13,355 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:13,355 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:13,356 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:13,356 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:15:13,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55466 deadline: 1732378573348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:13,356 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:13,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55464 deadline: 1732378573348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:13,356 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:13,356 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:13,356 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:15:13,356 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:13,356 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:13,356 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:15:13,357 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=90 2024-11-23T16:15:13,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66. 2024-11-23T16:15:13,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66. as already flushing 2024-11-23T16:15:13,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66. 2024-11-23T16:15:13,357 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] handler.RSProcedureHandler(58): pid=90 java.io.IOException: Unable to complete flush {ENCODED => 0b4ea01214996120537d91f0bab5da66, NAME => 'TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:15:13,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=90 java.io.IOException: Unable to complete flush {ENCODED => 0b4ea01214996120537d91f0bab5da66, NAME => 'TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:15:13,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.HMaster(4177): Remote procedure failed, pid=90 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0b4ea01214996120537d91f0bab5da66, NAME => 'TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0b4ea01214996120537d91f0bab5da66, NAME => 'TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:15:13,466 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=14, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/.tmp/A/c16c26c373a14e1fb105eedc59b9a723 2024-11-23T16:15:13,487 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/.tmp/B/8144d677a1c04286942a7fe81e4da595 is 50, key is test_row_0/B:col10/1732378512593/Put/seqid=0 2024-11-23T16:15:13,493 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742145_1321 (size=12001) 2024-11-23T16:15:13,510 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=90 2024-11-23T16:15:13,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66. 2024-11-23T16:15:13,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66. as already flushing 2024-11-23T16:15:13,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66. 2024-11-23T16:15:13,510 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] handler.RSProcedureHandler(58): pid=90 java.io.IOException: Unable to complete flush {ENCODED => 0b4ea01214996120537d91f0bab5da66, NAME => 'TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:15:13,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=90 java.io.IOException: Unable to complete flush {ENCODED => 0b4ea01214996120537d91f0bab5da66, NAME => 'TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:15:13,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.HMaster(4177): Remote procedure failed, pid=90 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0b4ea01214996120537d91f0bab5da66, NAME => 'TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0b4ea01214996120537d91f0bab5da66, NAME => 'TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:15:13,663 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=90 2024-11-23T16:15:13,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66. 2024-11-23T16:15:13,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66. as already flushing 2024-11-23T16:15:13,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66. 2024-11-23T16:15:13,663 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] handler.RSProcedureHandler(58): pid=90 java.io.IOException: Unable to complete flush {ENCODED => 0b4ea01214996120537d91f0bab5da66, NAME => 'TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:15:13,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=90 java.io.IOException: Unable to complete flush {ENCODED => 0b4ea01214996120537d91f0bab5da66, NAME => 'TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:15:13,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.HMaster(4177): Remote procedure failed, pid=90 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0b4ea01214996120537d91f0bab5da66, NAME => 'TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0b4ea01214996120537d91f0bab5da66, NAME => 'TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:15:13,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-23T16:15:13,815 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=90 2024-11-23T16:15:13,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66. 2024-11-23T16:15:13,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66. as already flushing 2024-11-23T16:15:13,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66. 2024-11-23T16:15:13,816 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] handler.RSProcedureHandler(58): pid=90 java.io.IOException: Unable to complete flush {ENCODED => 0b4ea01214996120537d91f0bab5da66, NAME => 'TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:15:13,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=90 java.io.IOException: Unable to complete flush {ENCODED => 0b4ea01214996120537d91f0bab5da66, NAME => 'TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:15:13,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.HMaster(4177): Remote procedure failed, pid=90 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0b4ea01214996120537d91f0bab5da66, NAME => 'TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0b4ea01214996120537d91f0bab5da66, NAME => 'TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:15:13,856 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:13,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55422 deadline: 1732378573848, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:13,857 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:13,857 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:13,857 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:15:13,865 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:13,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55412 deadline: 1732378573857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:13,865 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:13,865 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:13,866 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:15:13,879 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:13,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55404 deadline: 1732378573868, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:13,880 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:13,880 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:13,880 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:13,880 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:15:13,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55466 deadline: 1732378573868, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:13,880 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:13,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55464 deadline: 1732378573868, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:13,880 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:13,880 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:13,880 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:15:13,880 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:13,880 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:13,880 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:15:13,894 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/.tmp/B/8144d677a1c04286942a7fe81e4da595 2024-11-23T16:15:13,914 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/.tmp/C/7ad28d3a58e74aee953d1f7711776cb8 is 50, key is test_row_0/C:col10/1732378512593/Put/seqid=0 2024-11-23T16:15:13,918 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742146_1322 (size=12001) 2024-11-23T16:15:13,919 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/.tmp/C/7ad28d3a58e74aee953d1f7711776cb8 2024-11-23T16:15:13,923 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/.tmp/A/c16c26c373a14e1fb105eedc59b9a723 as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/A/c16c26c373a14e1fb105eedc59b9a723 2024-11-23T16:15:13,928 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/A/c16c26c373a14e1fb105eedc59b9a723, entries=150, sequenceid=14, filesize=30.2 K 2024-11-23T16:15:13,929 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/.tmp/B/8144d677a1c04286942a7fe81e4da595 as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/B/8144d677a1c04286942a7fe81e4da595 2024-11-23T16:15:13,932 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/B/8144d677a1c04286942a7fe81e4da595, entries=150, sequenceid=14, filesize=11.7 K 2024-11-23T16:15:13,934 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/.tmp/C/7ad28d3a58e74aee953d1f7711776cb8 as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/C/7ad28d3a58e74aee953d1f7711776cb8 2024-11-23T16:15:13,937 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/C/7ad28d3a58e74aee953d1f7711776cb8, entries=150, sequenceid=14, filesize=11.7 K 2024-11-23T16:15:13,938 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 0b4ea01214996120537d91f0bab5da66 in 1337ms, sequenceid=14, compaction requested=false 2024-11-23T16:15:13,938 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0b4ea01214996120537d91f0bab5da66: 2024-11-23T16:15:13,969 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=90 2024-11-23T16:15:13,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66. 2024-11-23T16:15:13,969 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2902): Flushing 0b4ea01214996120537d91f0bab5da66 3/3 column families, dataSize=134.18 KB heapSize=375.61 KB 2024-11-23T16:15:13,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0b4ea01214996120537d91f0bab5da66, store=A 2024-11-23T16:15:13,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-23T16:15:13,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0b4ea01214996120537d91f0bab5da66, store=B 2024-11-23T16:15:13,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-23T16:15:13,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0b4ea01214996120537d91f0bab5da66, store=C 2024-11-23T16:15:13,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-23T16:15:13,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241123659e4cf4a9934db9b2d5ecea34c83d45_0b4ea01214996120537d91f0bab5da66 is 50, key is test_row_0/A:col10/1732378512679/Put/seqid=0 2024-11-23T16:15:13,980 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742147_1323 (size=12154) 2024-11-23T16:15:13,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:13,984 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241123659e4cf4a9934db9b2d5ecea34c83d45_0b4ea01214996120537d91f0bab5da66 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123659e4cf4a9934db9b2d5ecea34c83d45_0b4ea01214996120537d91f0bab5da66 2024-11-23T16:15:13,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/.tmp/A/38d12e3abfb1412baa1c15149152447b, store: [table=TestAcidGuarantees family=A region=0b4ea01214996120537d91f0bab5da66] 2024-11-23T16:15:13,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/.tmp/A/38d12e3abfb1412baa1c15149152447b is 175, key is test_row_0/A:col10/1732378512679/Put/seqid=0 2024-11-23T16:15:13,988 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742148_1324 (size=30955) 2024-11-23T16:15:14,213 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-23T16:15:14,389 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=49, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/.tmp/A/38d12e3abfb1412baa1c15149152447b 2024-11-23T16:15:14,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/.tmp/B/2c9c68366e264b708b1a7b065ae4663e is 50, key is test_row_0/B:col10/1732378512679/Put/seqid=0 2024-11-23T16:15:14,401 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742149_1325 (size=12001) 2024-11-23T16:15:14,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-23T16:15:14,802 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/.tmp/B/2c9c68366e264b708b1a7b065ae4663e 2024-11-23T16:15:14,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/.tmp/C/ef5cf31c695d4057af66caabf14a5b5f is 50, key is test_row_0/C:col10/1732378512679/Put/seqid=0 2024-11-23T16:15:14,832 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742150_1326 (size=12001) 2024-11-23T16:15:14,833 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/.tmp/C/ef5cf31c695d4057af66caabf14a5b5f 2024-11-23T16:15:14,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/.tmp/A/38d12e3abfb1412baa1c15149152447b as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/A/38d12e3abfb1412baa1c15149152447b 2024-11-23T16:15:14,858 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/A/38d12e3abfb1412baa1c15149152447b, entries=150, sequenceid=49, filesize=30.2 K 2024-11-23T16:15:14,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/.tmp/B/2c9c68366e264b708b1a7b065ae4663e as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/B/2c9c68366e264b708b1a7b065ae4663e 2024-11-23T16:15:14,865 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/B/2c9c68366e264b708b1a7b065ae4663e, entries=150, sequenceid=49, filesize=11.7 K 2024-11-23T16:15:14,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/.tmp/C/ef5cf31c695d4057af66caabf14a5b5f as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/C/ef5cf31c695d4057af66caabf14a5b5f 2024-11-23T16:15:14,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,870 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/C/ef5cf31c695d4057af66caabf14a5b5f, entries=150, sequenceid=49, filesize=11.7 K 2024-11-23T16:15:14,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,871 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~375.56 KB/384576, currentSize=0 B/0 for 0b4ea01214996120537d91f0bab5da66 in 902ms, sequenceid=49, compaction requested=false 2024-11-23T16:15:14,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2603): Flush status journal for 0b4ea01214996120537d91f0bab5da66: 2024-11-23T16:15:14,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66. 2024-11-23T16:15:14,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=90 2024-11-23T16:15:14,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.HMaster(4169): Remote procedure done, pid=90 2024-11-23T16:15:14,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,874 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=90, resume processing ppid=89 2024-11-23T16:15:14,874 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=90, ppid=89, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2810 sec 2024-11-23T16:15:14,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,876 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=89, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=89, table=TestAcidGuarantees in 2.2870 sec 2024-11-23T16:15:14,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(8855): Flush requested on 0b4ea01214996120537d91f0bab5da66 2024-11-23T16:15:14,978 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0b4ea01214996120537d91f0bab5da66 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-23T16:15:14,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,978 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0b4ea01214996120537d91f0bab5da66, store=A 2024-11-23T16:15:14,978 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-23T16:15:14,978 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0b4ea01214996120537d91f0bab5da66, store=B 2024-11-23T16:15:14,978 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-23T16:15:14,978 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0b4ea01214996120537d91f0bab5da66, store=C 2024-11-23T16:15:14,978 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-23T16:15:14,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,989 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241123dbb3b61e507c4c4abe3dcdcc6667191e_0b4ea01214996120537d91f0bab5da66 is 50, key is test_row_0/A:col10/1732378514977/Put/seqid=0 2024-11-23T16:15:14,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:14,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:15,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:15,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:15,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:15,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:15,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:15,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:15,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:15,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:15,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:15,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:15,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:15,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:15,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:15,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:15,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:15,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:15,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:15,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:15,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:15,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:15,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:15,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:15,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:15,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:15,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:15,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:15,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:15,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:15,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:15,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:15,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:15,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:15,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:15,016 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742151_1327 (size=21918) 2024-11-23T16:15:15,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:15,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:15,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:15,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:15,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:15,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:15,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:15,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:15,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:15,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:15,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:15,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:15,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:15,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:15,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:15,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:15,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:15,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:15,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:15,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:15,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:15,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:15,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:15,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:15,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:15,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:15,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:15,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:15,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:15,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:15,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:15,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:15,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:15,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:15,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:15,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:15,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:15,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:15,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:15,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:15,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:15,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:15,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:15,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:15,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:15,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:15,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:15,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:15,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:15,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:15,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:15,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:15,058 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1a7c5470b472:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732378515025, max timestamp=1732378515055Num uniques 859; 2024-11-23T16:15:15,058 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1a7c5470b472:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732378515025, max timestamp=1732378515055Num uniques 859; 2024-11-23T16:15:15,058 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1a7c5470b472:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732378515025, max timestamp=1732378515055Num uniques 859; 2024-11-23T16:15:15,059 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1a7c5470b472:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-23T16:15:15,059 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1a7c5470b472:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-23T16:15:15,059 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1a7c5470b472:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-23T16:15:15,112 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:15,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55464 deadline: 1732378575096, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:15,113 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:15,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55404 deadline: 1732378575097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:15,113 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:15,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55466 deadline: 1732378575109, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:15,114 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:15,114 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:15,114 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:15:15,114 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:15,114 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:15,115 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:15:15,116 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1a7c5470b472:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1732378515094, max timestamp=1732378515111Num uniques 1018; 2024-11-23T16:15:15,116 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1a7c5470b472:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-11-23T16:15:15,116 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:15,116 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:15,116 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:15:15,116 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1a7c5470b472:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1732378515094, max timestamp=1732378515111Num uniques 1018; 2024-11-23T16:15:15,116 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1a7c5470b472:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1732378515094, max timestamp=1732378515111Num uniques 1018; 2024-11-23T16:15:15,117 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1a7c5470b472:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-11-23T16:15:15,117 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1a7c5470b472:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-11-23T16:15:15,215 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:15,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55412 deadline: 1732378575203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:15,215 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:15,215 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:15,215 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:15:15,219 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:15,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55422 deadline: 1732378575204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:15,219 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:15,219 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:15,219 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:15:15,234 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:15,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55404 deadline: 1732378575218, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:15,234 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:15,234 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:15,234 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:15,234 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:15:15,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55466 deadline: 1732378575218, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:15,234 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:15,234 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:15,234 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:15:15,235 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:15,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55464 deadline: 1732378575228, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:15,235 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:15,235 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:15,236 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:15:15,318 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:15,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55412 deadline: 1732378575318, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:15,319 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:15,319 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:15,319 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:15:15,337 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:15,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55422 deadline: 1732378575328, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:15,337 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:15,337 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:15,337 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:15:15,417 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:15,421 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241123dbb3b61e507c4c4abe3dcdcc6667191e_0b4ea01214996120537d91f0bab5da66 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123dbb3b61e507c4c4abe3dcdcc6667191e_0b4ea01214996120537d91f0bab5da66 2024-11-23T16:15:15,421 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/.tmp/A/0c2a554442fb42cd97b193b345e0b8b4, store: [table=TestAcidGuarantees family=A region=0b4ea01214996120537d91f0bab5da66] 2024-11-23T16:15:15,422 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/.tmp/A/0c2a554442fb42cd97b193b345e0b8b4 is 175, key is test_row_0/A:col10/1732378514977/Put/seqid=0 2024-11-23T16:15:15,426 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742152_1328 (size=65323) 2024-11-23T16:15:15,446 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:15,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55404 deadline: 1732378575438, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:15,447 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:15,447 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:15,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55466 deadline: 1732378575438, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:15,447 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:15,447 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:15:15,447 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:15,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55464 deadline: 1732378575438, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:15,447 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:15,447 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:15,447 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:15:15,448 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:15,448 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:15,448 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:15:15,531 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:15,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55412 deadline: 1732378575528, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:15,532 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:15,532 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:15,532 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:15:15,563 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:15,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55422 deadline: 1732378575548, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:15,563 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:15,563 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:15,563 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:15:15,771 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:15,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55404 deadline: 1732378575758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:15,771 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:15,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55466 deadline: 1732378575758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:15,772 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:15,772 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:15,772 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:15:15,772 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:15,772 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:15,772 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:15,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55464 deadline: 1732378575759, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:15,772 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:15:15,772 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:15,772 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:15,772 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:15:15,827 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=59, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/.tmp/A/0c2a554442fb42cd97b193b345e0b8b4 2024-11-23T16:15:15,834 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/.tmp/B/3cfdf5b8629c44b09f5bb64079e825e4 is 50, key is test_row_0/B:col10/1732378514977/Put/seqid=0 2024-11-23T16:15:15,839 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742153_1329 (size=12001) 2024-11-23T16:15:15,851 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:15,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55412 deadline: 1732378575838, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:15,851 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:15,851 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:15,851 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:15:15,881 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:15,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55422 deadline: 1732378575868, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:15,881 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:15,881 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:15,881 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:15:16,240 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/.tmp/B/3cfdf5b8629c44b09f5bb64079e825e4 2024-11-23T16:15:16,246 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/.tmp/C/1a071b2a0b464c38a2d197aba97025ed is 50, key is test_row_0/C:col10/1732378514977/Put/seqid=0 2024-11-23T16:15:16,250 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742154_1330 (size=12001) 2024-11-23T16:15:16,290 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:16,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55404 deadline: 1732378576278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:16,291 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:16,291 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:16,291 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:16,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55466 deadline: 1732378576278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:16,291 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:15:16,291 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:16,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55464 deadline: 1732378576278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:16,291 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:16,291 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:16,291 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:15:16,292 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:16,292 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:16,292 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:15:16,370 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:16,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55412 deadline: 1732378576358, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:16,370 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:16,370 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:16,370 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:15:16,400 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:16,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55422 deadline: 1732378576388, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:16,401 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:16,401 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:16,401 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:15:16,651 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/.tmp/C/1a071b2a0b464c38a2d197aba97025ed 2024-11-23T16:15:16,656 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/.tmp/A/0c2a554442fb42cd97b193b345e0b8b4 as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/A/0c2a554442fb42cd97b193b345e0b8b4 2024-11-23T16:15:16,660 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/A/0c2a554442fb42cd97b193b345e0b8b4, entries=350, sequenceid=59, filesize=63.8 K 2024-11-23T16:15:16,661 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/.tmp/B/3cfdf5b8629c44b09f5bb64079e825e4 as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/B/3cfdf5b8629c44b09f5bb64079e825e4 2024-11-23T16:15:16,665 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/B/3cfdf5b8629c44b09f5bb64079e825e4, entries=150, sequenceid=59, filesize=11.7 K 2024-11-23T16:15:16,665 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/.tmp/C/1a071b2a0b464c38a2d197aba97025ed as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/C/1a071b2a0b464c38a2d197aba97025ed 2024-11-23T16:15:16,668 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/C/1a071b2a0b464c38a2d197aba97025ed, entries=150, sequenceid=59, filesize=11.7 K 2024-11-23T16:15:16,669 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 0b4ea01214996120537d91f0bab5da66 in 1691ms, sequenceid=59, compaction requested=true 2024-11-23T16:15:16,669 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0b4ea01214996120537d91f0bab5da66: 2024-11-23T16:15:16,669 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0b4ea01214996120537d91f0bab5da66:A, priority=-2147483648, current under compaction store size is 1 2024-11-23T16:15:16,669 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T16:15:16,670 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T16:15:16,670 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0b4ea01214996120537d91f0bab5da66:B, priority=-2147483648, current under compaction store size is 2 2024-11-23T16:15:16,670 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T16:15:16,670 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0b4ea01214996120537d91f0bab5da66:C, priority=-2147483648, current under compaction store size is 3 2024-11-23T16:15:16,670 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T16:15:16,670 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T16:15:16,670 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T16:15:16,671 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 127233 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T16:15:16,671 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HStore(1541): 0b4ea01214996120537d91f0bab5da66/B is initiating minor compaction (all files) 2024-11-23T16:15:16,671 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0b4ea01214996120537d91f0bab5da66/B in TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66. 2024-11-23T16:15:16,671 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HStore(1541): 0b4ea01214996120537d91f0bab5da66/A is initiating minor compaction (all files) 2024-11-23T16:15:16,671 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0b4ea01214996120537d91f0bab5da66/A in TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66. 2024-11-23T16:15:16,671 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/B/8144d677a1c04286942a7fe81e4da595, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/B/2c9c68366e264b708b1a7b065ae4663e, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/B/3cfdf5b8629c44b09f5bb64079e825e4] into tmpdir=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/.tmp, totalSize=35.2 K 2024-11-23T16:15:16,671 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/A/c16c26c373a14e1fb105eedc59b9a723, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/A/38d12e3abfb1412baa1c15149152447b, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/A/0c2a554442fb42cd97b193b345e0b8b4] into tmpdir=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/.tmp, totalSize=124.3 K 2024-11-23T16:15:16,671 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66. 2024-11-23T16:15:16,671 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66. files: [hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/A/c16c26c373a14e1fb105eedc59b9a723, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/A/38d12e3abfb1412baa1c15149152447b, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/A/0c2a554442fb42cd97b193b345e0b8b4] 2024-11-23T16:15:16,671 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.Compactor(225): Compacting 8144d677a1c04286942a7fe81e4da595, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1732378512593 2024-11-23T16:15:16,671 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.Compactor(225): Compacting c16c26c373a14e1fb105eedc59b9a723, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1732378512593 2024-11-23T16:15:16,672 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.Compactor(225): Compacting 2c9c68366e264b708b1a7b065ae4663e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1732378512679 2024-11-23T16:15:16,672 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.Compactor(225): Compacting 38d12e3abfb1412baa1c15149152447b, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1732378512679 2024-11-23T16:15:16,672 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.Compactor(225): Compacting 3cfdf5b8629c44b09f5bb64079e825e4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1732378514958 2024-11-23T16:15:16,672 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0c2a554442fb42cd97b193b345e0b8b4, keycount=350, bloomtype=ROW, size=63.8 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1732378514956 2024-11-23T16:15:16,678 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=0b4ea01214996120537d91f0bab5da66] 2024-11-23T16:15:16,680 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0b4ea01214996120537d91f0bab5da66#B#compaction#279 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-23T16:15:16,680 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411234452135536384872873f551d1a0f9915_0b4ea01214996120537d91f0bab5da66 store=[table=TestAcidGuarantees family=A region=0b4ea01214996120537d91f0bab5da66] 2024-11-23T16:15:16,681 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/.tmp/B/5dbd82b35f6b4e58b22e846d1dc0a6c9 is 50, key is test_row_0/B:col10/1732378514977/Put/seqid=0 2024-11-23T16:15:16,682 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411234452135536384872873f551d1a0f9915_0b4ea01214996120537d91f0bab5da66, store=[table=TestAcidGuarantees family=A region=0b4ea01214996120537d91f0bab5da66] 2024-11-23T16:15:16,682 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411234452135536384872873f551d1a0f9915_0b4ea01214996120537d91f0bab5da66 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=0b4ea01214996120537d91f0bab5da66] 2024-11-23T16:15:16,690 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742155_1331 (size=12104) 2024-11-23T16:15:16,698 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742156_1332 (size=4469) 2024-11-23T16:15:16,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-23T16:15:16,748 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-23T16:15:16,748 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-23T16:15:16,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] procedure2.ProcedureExecutor(1139): Stored pid=91, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=91, table=TestAcidGuarantees 2024-11-23T16:15:16,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-23T16:15:16,751 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=91, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=91, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-23T16:15:16,751 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=91, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=91, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-23T16:15:16,751 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=92, ppid=91, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-23T16:15:16,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-23T16:15:16,904 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-11-23T16:15:16,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66. 2024-11-23T16:15:16,904 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2902): Flushing 0b4ea01214996120537d91f0bab5da66 3/3 column families, dataSize=134.18 KB heapSize=379.45 KB 2024-11-23T16:15:16,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0b4ea01214996120537d91f0bab5da66, store=A 2024-11-23T16:15:16,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-23T16:15:16,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0b4ea01214996120537d91f0bab5da66, store=B 2024-11-23T16:15:16,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-23T16:15:16,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0b4ea01214996120537d91f0bab5da66, store=C 2024-11-23T16:15:16,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-23T16:15:16,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241123ad776715dfe64847ac99d06475270914_0b4ea01214996120537d91f0bab5da66 is 50, key is test_row_0/A:col10/1732378515201/Put/seqid=0 2024-11-23T16:15:16,976 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742157_1333 (size=12154) 2024-11-23T16:15:17,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-23T16:15:17,098 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/.tmp/B/5dbd82b35f6b4e58b22e846d1dc0a6c9 as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/B/5dbd82b35f6b4e58b22e846d1dc0a6c9 2024-11-23T16:15:17,100 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0b4ea01214996120537d91f0bab5da66#A#compaction#280 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T16:15:17,101 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/.tmp/A/95a56319c3994539be6585dc07758f69 is 175, key is test_row_0/A:col10/1732378514977/Put/seqid=0 2024-11-23T16:15:17,105 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0b4ea01214996120537d91f0bab5da66/B of 0b4ea01214996120537d91f0bab5da66 into 5dbd82b35f6b4e58b22e846d1dc0a6c9(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T16:15:17,105 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0b4ea01214996120537d91f0bab5da66: 2024-11-23T16:15:17,105 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., storeName=0b4ea01214996120537d91f0bab5da66/B, priority=13, startTime=1732378516669; duration=0sec 2024-11-23T16:15:17,106 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T16:15:17,106 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0b4ea01214996120537d91f0bab5da66:B 2024-11-23T16:15:17,106 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T16:15:17,107 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T16:15:17,107 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HStore(1541): 0b4ea01214996120537d91f0bab5da66/C is initiating minor compaction (all files) 2024-11-23T16:15:17,107 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0b4ea01214996120537d91f0bab5da66/C in TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66. 2024-11-23T16:15:17,107 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/C/7ad28d3a58e74aee953d1f7711776cb8, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/C/ef5cf31c695d4057af66caabf14a5b5f, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/C/1a071b2a0b464c38a2d197aba97025ed] into tmpdir=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/.tmp, totalSize=35.2 K 2024-11-23T16:15:17,107 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.Compactor(225): Compacting 7ad28d3a58e74aee953d1f7711776cb8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1732378512593 2024-11-23T16:15:17,107 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.Compactor(225): Compacting ef5cf31c695d4057af66caabf14a5b5f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1732378512679 2024-11-23T16:15:17,108 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.Compactor(225): Compacting 1a071b2a0b464c38a2d197aba97025ed, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1732378514958 2024-11-23T16:15:17,118 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742158_1334 (size=31058) 2024-11-23T16:15:17,132 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0b4ea01214996120537d91f0bab5da66#C#compaction#282 average throughput is 1.09 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T16:15:17,133 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/.tmp/C/2d663f8abde74ba9b95108466a94a376 is 50, key is test_row_0/C:col10/1732378514977/Put/seqid=0 2024-11-23T16:15:17,139 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/.tmp/A/95a56319c3994539be6585dc07758f69 as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/A/95a56319c3994539be6585dc07758f69 2024-11-23T16:15:17,153 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0b4ea01214996120537d91f0bab5da66/A of 0b4ea01214996120537d91f0bab5da66 into 95a56319c3994539be6585dc07758f69(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T16:15:17,153 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0b4ea01214996120537d91f0bab5da66: 2024-11-23T16:15:17,153 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., storeName=0b4ea01214996120537d91f0bab5da66/A, priority=13, startTime=1732378516669; duration=0sec 2024-11-23T16:15:17,154 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T16:15:17,154 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0b4ea01214996120537d91f0bab5da66:A 2024-11-23T16:15:17,190 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742159_1335 (size=12104) 2024-11-23T16:15:17,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(8855): Flush requested on 0b4ea01214996120537d91f0bab5da66 2024-11-23T16:15:17,305 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66. as already flushing 2024-11-23T16:15:17,335 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:17,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55466 deadline: 1732378577333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:17,336 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:17,336 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:17,336 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:15:17,350 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:17,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55464 deadline: 1732378577335, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:17,350 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:17,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55404 deadline: 1732378577335, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:17,351 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:17,351 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:17,351 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:15:17,352 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:17,352 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:17,352 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:15:17,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:17,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-23T16:15:17,381 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241123ad776715dfe64847ac99d06475270914_0b4ea01214996120537d91f0bab5da66 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123ad776715dfe64847ac99d06475270914_0b4ea01214996120537d91f0bab5da66 2024-11-23T16:15:17,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/.tmp/A/5abdcf514da744d4b5e863ac23d34c04, store: [table=TestAcidGuarantees family=A region=0b4ea01214996120537d91f0bab5da66] 2024-11-23T16:15:17,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/.tmp/A/5abdcf514da744d4b5e863ac23d34c04 is 175, key is test_row_0/A:col10/1732378515201/Put/seqid=0 2024-11-23T16:15:17,387 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:17,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55412 deadline: 1732378577378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:17,387 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 1a7c5470b472,44715,1732378420375 for 'test_row_1' in 0b4ea01214996120537d91f0bab5da66 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2185 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-23T16:15:17,388 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742160_1336 (size=30955) 2024-11-23T16:15:17,388 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:17,388 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:17,388 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:15:17,420 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:17,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55422 deadline: 1732378577408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:17,421 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 1a7c5470b472,44715,1732378420375 for 'test_row_0' in 0b4ea01214996120537d91f0bab5da66 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2218 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-23T16:15:17,422 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:17,422 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:17,422 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:15:17,450 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:17,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55466 deadline: 1732378577439, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:17,451 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:17,451 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:17,451 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:15:17,471 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:17,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55464 deadline: 1732378577458, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:17,472 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:17,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55404 deadline: 1732378577458, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:17,474 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:17,474 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:17,474 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:15:17,474 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:17,474 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:17,474 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:15:17,597 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/.tmp/C/2d663f8abde74ba9b95108466a94a376 as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/C/2d663f8abde74ba9b95108466a94a376 2024-11-23T16:15:17,605 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0b4ea01214996120537d91f0bab5da66/C of 0b4ea01214996120537d91f0bab5da66 into 2d663f8abde74ba9b95108466a94a376(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T16:15:17,605 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0b4ea01214996120537d91f0bab5da66: 2024-11-23T16:15:17,605 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., storeName=0b4ea01214996120537d91f0bab5da66/C, priority=13, startTime=1732378516670; duration=0sec 2024-11-23T16:15:17,605 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T16:15:17,605 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0b4ea01214996120537d91f0bab5da66:C 2024-11-23T16:15:17,669 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:17,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55466 deadline: 1732378577658, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:17,669 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:17,669 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:17,669 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:15:17,685 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:17,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55464 deadline: 1732378577678, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:17,686 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:17,686 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:17,686 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:17,686 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:15:17,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55404 deadline: 1732378577679, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:17,686 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:17,686 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:17,686 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:15:17,788 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=112, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/.tmp/A/5abdcf514da744d4b5e863ac23d34c04 2024-11-23T16:15:17,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/.tmp/B/e17e820c42e3429cbd7ade2eb197b91a is 50, key is test_row_0/B:col10/1732378515201/Put/seqid=0 2024-11-23T16:15:17,833 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742161_1337 (size=12001) 2024-11-23T16:15:17,834 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=112 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/.tmp/B/e17e820c42e3429cbd7ade2eb197b91a 2024-11-23T16:15:17,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/.tmp/C/3665238e9bb74ed0b90676dfdcb39625 is 50, key is test_row_0/C:col10/1732378515201/Put/seqid=0 2024-11-23T16:15:17,848 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742162_1338 (size=12001) 2024-11-23T16:15:17,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-23T16:15:17,991 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:17,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55466 deadline: 1732378577977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:17,992 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:17,992 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:17,992 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:15:17,998 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:17,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55464 deadline: 1732378577988, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:17,998 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:17,999 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:17,999 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:15:18,006 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:18,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55404 deadline: 1732378577998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:18,006 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:18,006 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:18,006 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:15:18,249 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=112 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/.tmp/C/3665238e9bb74ed0b90676dfdcb39625 2024-11-23T16:15:18,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/.tmp/A/5abdcf514da744d4b5e863ac23d34c04 as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/A/5abdcf514da744d4b5e863ac23d34c04 2024-11-23T16:15:18,265 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/A/5abdcf514da744d4b5e863ac23d34c04, entries=150, sequenceid=112, filesize=30.2 K 2024-11-23T16:15:18,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/.tmp/B/e17e820c42e3429cbd7ade2eb197b91a as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/B/e17e820c42e3429cbd7ade2eb197b91a 2024-11-23T16:15:18,279 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/B/e17e820c42e3429cbd7ade2eb197b91a, entries=150, sequenceid=112, filesize=11.7 K 2024-11-23T16:15:18,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/.tmp/C/3665238e9bb74ed0b90676dfdcb39625 as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/C/3665238e9bb74ed0b90676dfdcb39625 2024-11-23T16:15:18,284 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/C/3665238e9bb74ed0b90676dfdcb39625, entries=150, sequenceid=112, filesize=11.7 K 2024-11-23T16:15:18,291 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~379.41 KB/388512, currentSize=53.67 KB/54960 for 0b4ea01214996120537d91f0bab5da66 in 1387ms, sequenceid=112, compaction requested=false 2024-11-23T16:15:18,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2603): Flush status journal for 0b4ea01214996120537d91f0bab5da66: 2024-11-23T16:15:18,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66. 2024-11-23T16:15:18,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=92 2024-11-23T16:15:18,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.HMaster(4169): Remote procedure done, pid=92 2024-11-23T16:15:18,295 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=92, resume processing ppid=91 2024-11-23T16:15:18,295 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=92, ppid=91, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5420 sec 2024-11-23T16:15:18,297 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=91, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=91, table=TestAcidGuarantees in 1.5470 sec 2024-11-23T16:15:18,507 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0b4ea01214996120537d91f0bab5da66 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-23T16:15:18,507 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0b4ea01214996120537d91f0bab5da66, store=A 2024-11-23T16:15:18,507 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-23T16:15:18,507 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0b4ea01214996120537d91f0bab5da66, store=B 2024-11-23T16:15:18,507 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-23T16:15:18,507 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0b4ea01214996120537d91f0bab5da66, store=C 2024-11-23T16:15:18,507 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-23T16:15:18,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(8855): Flush requested on 0b4ea01214996120537d91f0bab5da66 2024-11-23T16:15:18,535 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112317bbd387ba2749e9bf4c050a05c44f4c_0b4ea01214996120537d91f0bab5da66 is 50, key is test_row_0/A:col10/1732378517333/Put/seqid=0 2024-11-23T16:15:18,579 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742163_1339 (size=14594) 2024-11-23T16:15:18,584 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:18,588 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112317bbd387ba2749e9bf4c050a05c44f4c_0b4ea01214996120537d91f0bab5da66 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112317bbd387ba2749e9bf4c050a05c44f4c_0b4ea01214996120537d91f0bab5da66 2024-11-23T16:15:18,589 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/.tmp/A/26562298cc40485c858fbbc4359a1a8d, store: [table=TestAcidGuarantees family=A region=0b4ea01214996120537d91f0bab5da66] 2024-11-23T16:15:18,590 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/.tmp/A/26562298cc40485c858fbbc4359a1a8d is 175, key is test_row_0/A:col10/1732378517333/Put/seqid=0 2024-11-23T16:15:18,636 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742164_1340 (size=39549) 2024-11-23T16:15:18,636 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:18,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55464 deadline: 1732378578618, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:18,636 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:18,637 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:18,636 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:18,637 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:15:18,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55466 deadline: 1732378578619, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:18,637 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:18,637 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:18,637 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:15:18,637 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:18,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55404 deadline: 1732378578621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:18,638 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:18,638 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:18,638 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:15:18,759 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:18,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55466 deadline: 1732378578748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:18,760 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:18,760 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:18,760 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:18,760 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:15:18,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55404 deadline: 1732378578748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:18,760 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:18,761 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:18,761 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:18,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55464 deadline: 1732378578748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:18,761 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:15:18,761 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:18,761 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:18,761 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:15:18,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-23T16:15:18,898 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-23T16:15:18,898 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-23T16:15:18,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] procedure2.ProcedureExecutor(1139): Stored pid=93, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees 2024-11-23T16:15:18,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-23T16:15:18,900 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=93, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-23T16:15:18,901 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=93, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-23T16:15:18,901 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=94, ppid=93, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-23T16:15:18,985 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:18,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55404 deadline: 1732378578968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:18,986 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:18,986 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:18,986 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:18,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55464 deadline: 1732378578969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:18,986 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:15:18,986 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:18,986 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:18,986 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:15:18,986 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:18,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55466 deadline: 1732378578969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:18,987 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:18,987 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:18,987 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:15:19,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-23T16:15:19,037 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=127, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/.tmp/A/26562298cc40485c858fbbc4359a1a8d 2024-11-23T16:15:19,050 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/.tmp/B/144a47b3eac943cc96fdb9f169a47d6c is 50, key is test_row_0/B:col10/1732378517333/Put/seqid=0 2024-11-23T16:15:19,054 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-23T16:15:19,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66. 2024-11-23T16:15:19,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66. as already flushing 2024-11-23T16:15:19,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66. 2024-11-23T16:15:19,054 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 0b4ea01214996120537d91f0bab5da66, NAME => 'TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:15:19,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 0b4ea01214996120537d91f0bab5da66, NAME => 'TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:15:19,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0b4ea01214996120537d91f0bab5da66, NAME => 'TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0b4ea01214996120537d91f0bab5da66, NAME => 'TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:15:19,075 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742165_1341 (size=12001) 2024-11-23T16:15:19,078 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=127 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/.tmp/B/144a47b3eac943cc96fdb9f169a47d6c 2024-11-23T16:15:19,100 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/.tmp/C/24d7117c8c274bbc90bfdd548d9714bb is 50, key is test_row_0/C:col10/1732378517333/Put/seqid=0 2024-11-23T16:15:19,109 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742166_1342 (size=12001) 2024-11-23T16:15:19,110 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=127 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/.tmp/C/24d7117c8c274bbc90bfdd548d9714bb 2024-11-23T16:15:19,116 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/.tmp/A/26562298cc40485c858fbbc4359a1a8d as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/A/26562298cc40485c858fbbc4359a1a8d 2024-11-23T16:15:19,121 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/A/26562298cc40485c858fbbc4359a1a8d, entries=200, sequenceid=127, filesize=38.6 K 2024-11-23T16:15:19,122 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/.tmp/B/144a47b3eac943cc96fdb9f169a47d6c as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/B/144a47b3eac943cc96fdb9f169a47d6c 2024-11-23T16:15:19,128 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/B/144a47b3eac943cc96fdb9f169a47d6c, entries=150, sequenceid=127, filesize=11.7 K 2024-11-23T16:15:19,129 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/.tmp/C/24d7117c8c274bbc90bfdd548d9714bb as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/C/24d7117c8c274bbc90bfdd548d9714bb 2024-11-23T16:15:19,133 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/C/24d7117c8c274bbc90bfdd548d9714bb, entries=150, sequenceid=127, filesize=11.7 K 2024-11-23T16:15:19,134 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 0b4ea01214996120537d91f0bab5da66 in 628ms, sequenceid=127, compaction requested=true 2024-11-23T16:15:19,134 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0b4ea01214996120537d91f0bab5da66: 2024-11-23T16:15:19,134 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0b4ea01214996120537d91f0bab5da66:A, priority=-2147483648, current under compaction store size is 1 2024-11-23T16:15:19,134 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T16:15:19,134 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T16:15:19,138 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T16:15:19,138 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0b4ea01214996120537d91f0bab5da66:B, priority=-2147483648, current under compaction store size is 2 2024-11-23T16:15:19,138 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T16:15:19,138 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0b4ea01214996120537d91f0bab5da66:C, priority=-2147483648, current under compaction store size is 3 2024-11-23T16:15:19,138 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T16:15:19,139 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101562 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T16:15:19,139 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HStore(1541): 0b4ea01214996120537d91f0bab5da66/A is initiating minor compaction (all files) 2024-11-23T16:15:19,139 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0b4ea01214996120537d91f0bab5da66/A in TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66. 2024-11-23T16:15:19,139 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/A/95a56319c3994539be6585dc07758f69, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/A/5abdcf514da744d4b5e863ac23d34c04, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/A/26562298cc40485c858fbbc4359a1a8d] into tmpdir=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/.tmp, totalSize=99.2 K 2024-11-23T16:15:19,139 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66. 2024-11-23T16:15:19,139 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66. files: [hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/A/95a56319c3994539be6585dc07758f69, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/A/5abdcf514da744d4b5e863ac23d34c04, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/A/26562298cc40485c858fbbc4359a1a8d] 2024-11-23T16:15:19,140 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T16:15:19,140 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HStore(1541): 0b4ea01214996120537d91f0bab5da66/B is initiating minor compaction (all files) 2024-11-23T16:15:19,140 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0b4ea01214996120537d91f0bab5da66/B in TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66. 2024-11-23T16:15:19,141 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/B/5dbd82b35f6b4e58b22e846d1dc0a6c9, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/B/e17e820c42e3429cbd7ade2eb197b91a, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/B/144a47b3eac943cc96fdb9f169a47d6c] into tmpdir=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/.tmp, totalSize=35.3 K 2024-11-23T16:15:19,142 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.Compactor(225): Compacting 95a56319c3994539be6585dc07758f69, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1732378514958 2024-11-23T16:15:19,143 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.Compactor(225): Compacting 5dbd82b35f6b4e58b22e846d1dc0a6c9, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1732378514958 2024-11-23T16:15:19,143 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5abdcf514da744d4b5e863ac23d34c04, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=112, earliestPutTs=1732378515184 2024-11-23T16:15:19,143 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.Compactor(225): Compacting e17e820c42e3429cbd7ade2eb197b91a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=112, earliestPutTs=1732378515184 2024-11-23T16:15:19,143 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.Compactor(225): Compacting 26562298cc40485c858fbbc4359a1a8d, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=127, earliestPutTs=1732378517318 2024-11-23T16:15:19,144 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.Compactor(225): Compacting 144a47b3eac943cc96fdb9f169a47d6c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=127, earliestPutTs=1732378517333 2024-11-23T16:15:19,169 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=0b4ea01214996120537d91f0bab5da66] 2024-11-23T16:15:19,172 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0b4ea01214996120537d91f0bab5da66#B#compaction#289 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-23T16:15:19,173 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/.tmp/B/0e7a82f77905450992d2e76e9b85cd30 is 50, key is test_row_0/B:col10/1732378517333/Put/seqid=0 2024-11-23T16:15:19,184 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241123acd0104e76ff4fdca1fc3cc8c9ff4108_0b4ea01214996120537d91f0bab5da66 store=[table=TestAcidGuarantees family=A region=0b4ea01214996120537d91f0bab5da66] 2024-11-23T16:15:19,186 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241123acd0104e76ff4fdca1fc3cc8c9ff4108_0b4ea01214996120537d91f0bab5da66, store=[table=TestAcidGuarantees family=A region=0b4ea01214996120537d91f0bab5da66] 2024-11-23T16:15:19,186 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241123acd0104e76ff4fdca1fc3cc8c9ff4108_0b4ea01214996120537d91f0bab5da66 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=0b4ea01214996120537d91f0bab5da66] 2024-11-23T16:15:19,196 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742167_1343 (size=12207) 2024-11-23T16:15:19,203 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742168_1344 (size=4469) 2024-11-23T16:15:19,204 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0b4ea01214996120537d91f0bab5da66#A#compaction#288 average throughput is 0.70 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T16:15:19,205 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/.tmp/A/45d9fc022931492fb4f0e113a3584846 is 175, key is test_row_0/A:col10/1732378517333/Put/seqid=0 2024-11-23T16:15:19,206 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-23T16:15:19,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66. 2024-11-23T16:15:19,207 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2902): Flushing 0b4ea01214996120537d91f0bab5da66 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-23T16:15:19,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0b4ea01214996120537d91f0bab5da66, store=A 2024-11-23T16:15:19,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-23T16:15:19,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0b4ea01214996120537d91f0bab5da66, store=B 2024-11-23T16:15:19,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-23T16:15:19,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0b4ea01214996120537d91f0bab5da66, store=C 2024-11-23T16:15:19,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-23T16:15:19,212 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/.tmp/B/0e7a82f77905450992d2e76e9b85cd30 as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/B/0e7a82f77905450992d2e76e9b85cd30 2024-11-23T16:15:19,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-23T16:15:19,228 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0b4ea01214996120537d91f0bab5da66/B of 0b4ea01214996120537d91f0bab5da66 into 0e7a82f77905450992d2e76e9b85cd30(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T16:15:19,228 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0b4ea01214996120537d91f0bab5da66: 2024-11-23T16:15:19,228 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., storeName=0b4ea01214996120537d91f0bab5da66/B, priority=13, startTime=1732378519134; duration=0sec 2024-11-23T16:15:19,228 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T16:15:19,228 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0b4ea01214996120537d91f0bab5da66:B 2024-11-23T16:15:19,228 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T16:15:19,238 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T16:15:19,238 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HStore(1541): 0b4ea01214996120537d91f0bab5da66/C is initiating minor compaction (all files) 2024-11-23T16:15:19,238 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0b4ea01214996120537d91f0bab5da66/C in TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66. 2024-11-23T16:15:19,238 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/C/2d663f8abde74ba9b95108466a94a376, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/C/3665238e9bb74ed0b90676dfdcb39625, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/C/24d7117c8c274bbc90bfdd548d9714bb] into tmpdir=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/.tmp, totalSize=35.3 K 2024-11-23T16:15:19,239 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.Compactor(225): Compacting 2d663f8abde74ba9b95108466a94a376, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1732378514958 2024-11-23T16:15:19,239 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.Compactor(225): Compacting 3665238e9bb74ed0b90676dfdcb39625, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=112, earliestPutTs=1732378515184 2024-11-23T16:15:19,240 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.Compactor(225): Compacting 24d7117c8c274bbc90bfdd548d9714bb, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=127, earliestPutTs=1732378517333 2024-11-23T16:15:19,248 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742169_1345 (size=31161) 2024-11-23T16:15:19,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411230a5a65e2e1e741f4b85db6f997ef688e_0b4ea01214996120537d91f0bab5da66 is 50, key is test_row_0/A:col10/1732378518618/Put/seqid=0 2024-11-23T16:15:19,254 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0b4ea01214996120537d91f0bab5da66#C#compaction#291 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T16:15:19,255 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/.tmp/C/0aa96d0e877843d6a90e2557993e71b5 is 50, key is test_row_0/C:col10/1732378517333/Put/seqid=0 2024-11-23T16:15:19,266 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/.tmp/A/45d9fc022931492fb4f0e113a3584846 as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/A/45d9fc022931492fb4f0e113a3584846 2024-11-23T16:15:19,272 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0b4ea01214996120537d91f0bab5da66/A of 0b4ea01214996120537d91f0bab5da66 into 45d9fc022931492fb4f0e113a3584846(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T16:15:19,272 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0b4ea01214996120537d91f0bab5da66: 2024-11-23T16:15:19,272 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., storeName=0b4ea01214996120537d91f0bab5da66/A, priority=13, startTime=1732378519134; duration=0sec 2024-11-23T16:15:19,272 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T16:15:19,272 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0b4ea01214996120537d91f0bab5da66:A 2024-11-23T16:15:19,301 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742170_1346 (size=12304) 2024-11-23T16:15:19,305 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66. as already flushing 2024-11-23T16:15:19,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(8855): Flush requested on 0b4ea01214996120537d91f0bab5da66 2024-11-23T16:15:19,313 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742171_1347 (size=12207) 2024-11-23T16:15:19,323 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/.tmp/C/0aa96d0e877843d6a90e2557993e71b5 as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/C/0aa96d0e877843d6a90e2557993e71b5 2024-11-23T16:15:19,329 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0b4ea01214996120537d91f0bab5da66/C of 0b4ea01214996120537d91f0bab5da66 into 0aa96d0e877843d6a90e2557993e71b5(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T16:15:19,329 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0b4ea01214996120537d91f0bab5da66: 2024-11-23T16:15:19,329 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., storeName=0b4ea01214996120537d91f0bab5da66/C, priority=13, startTime=1732378519138; duration=0sec 2024-11-23T16:15:19,329 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T16:15:19,329 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0b4ea01214996120537d91f0bab5da66:C 2024-11-23T16:15:19,349 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:19,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55464 deadline: 1732378579330, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:19,350 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:19,350 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:19,350 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:15:19,365 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:19,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55404 deadline: 1732378579344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:19,366 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:19,366 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:19,366 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:19,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55466 deadline: 1732378579344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:19,366 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:15:19,366 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:19,366 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:19,366 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:15:19,423 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:19,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55412 deadline: 1732378579408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:19,423 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 1a7c5470b472,44715,1732378420375 for 'test_row_1' in 0b4ea01214996120537d91f0bab5da66 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4221 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-23T16:15:19,424 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:19,424 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:19,424 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:15:19,459 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:19,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55422 deadline: 1732378579448, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:19,459 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 1a7c5470b472,44715,1732378420375 for 'test_row_0' in 0b4ea01214996120537d91f0bab5da66 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4255 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-23T16:15:19,460 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:19,460 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:19,460 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:15:19,467 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:19,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55464 deadline: 1732378579458, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:19,468 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:19,468 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:19,468 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:15:19,479 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:19,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55466 deadline: 1732378579468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:19,479 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:19,480 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:19,480 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:15:19,490 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:19,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55404 deadline: 1732378579478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:19,493 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:19,494 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:19,494 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:15:19,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-23T16:15:19,687 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:19,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55464 deadline: 1732378579677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:19,687 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:19,687 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:19,687 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:15:19,690 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:19,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55466 deadline: 1732378579689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:19,690 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:19,690 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:19,690 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:15:19,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:19,704 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:19,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55404 deadline: 1732378579698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:19,705 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:19,705 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:19,705 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:15:19,706 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411230a5a65e2e1e741f4b85db6f997ef688e_0b4ea01214996120537d91f0bab5da66 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411230a5a65e2e1e741f4b85db6f997ef688e_0b4ea01214996120537d91f0bab5da66 2024-11-23T16:15:19,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/.tmp/A/4b4d8e58d563499c8442ce94d2dac1c8, store: [table=TestAcidGuarantees family=A region=0b4ea01214996120537d91f0bab5da66] 2024-11-23T16:15:19,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/.tmp/A/4b4d8e58d563499c8442ce94d2dac1c8 is 175, key is test_row_0/A:col10/1732378518618/Put/seqid=0 2024-11-23T16:15:19,730 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742172_1348 (size=31105) 2024-11-23T16:15:20,005 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:20,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55464 deadline: 1732378579998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:20,005 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:20,005 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:20,005 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:20,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55466 deadline: 1732378579998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:20,005 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:15:20,005 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:20,005 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:20,005 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:15:20,015 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:20,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55404 deadline: 1732378580008, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:20,016 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:20,016 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:20,016 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:15:20,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-23T16:15:20,108 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-23T16:15:20,108 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-23T16:15:20,130 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=147, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/.tmp/A/4b4d8e58d563499c8442ce94d2dac1c8 2024-11-23T16:15:20,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/.tmp/B/00f659f3e9d44b2cb9249f39840dc25a is 50, key is test_row_0/B:col10/1732378518618/Put/seqid=0 2024-11-23T16:15:20,141 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742173_1349 (size=12151) 2024-11-23T16:15:20,517 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:20,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55466 deadline: 1732378580507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:20,518 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:20,518 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:20,518 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:15:20,524 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:20,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55464 deadline: 1732378580518, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:20,524 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:20,524 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:20,524 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:15:20,537 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:20,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55404 deadline: 1732378580528, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:20,538 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:20,538 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:20,538 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:15:20,542 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=147 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/.tmp/B/00f659f3e9d44b2cb9249f39840dc25a 2024-11-23T16:15:20,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/.tmp/C/d9293febc9d449adb396df71274727ee is 50, key is test_row_0/C:col10/1732378518618/Put/seqid=0 2024-11-23T16:15:20,553 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742174_1350 (size=12151) 2024-11-23T16:15:20,954 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=147 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/.tmp/C/d9293febc9d449adb396df71274727ee 2024-11-23T16:15:20,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/.tmp/A/4b4d8e58d563499c8442ce94d2dac1c8 as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/A/4b4d8e58d563499c8442ce94d2dac1c8 2024-11-23T16:15:20,962 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/A/4b4d8e58d563499c8442ce94d2dac1c8, entries=150, sequenceid=147, filesize=30.4 K 2024-11-23T16:15:20,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/.tmp/B/00f659f3e9d44b2cb9249f39840dc25a as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/B/00f659f3e9d44b2cb9249f39840dc25a 2024-11-23T16:15:20,966 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/B/00f659f3e9d44b2cb9249f39840dc25a, entries=150, sequenceid=147, filesize=11.9 K 2024-11-23T16:15:20,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/.tmp/C/d9293febc9d449adb396df71274727ee as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/C/d9293febc9d449adb396df71274727ee 2024-11-23T16:15:20,970 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/C/d9293febc9d449adb396df71274727ee, entries=150, sequenceid=147, filesize=11.9 K 2024-11-23T16:15:20,971 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 0b4ea01214996120537d91f0bab5da66 in 1764ms, sequenceid=147, compaction requested=false 2024-11-23T16:15:20,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2603): Flush status journal for 0b4ea01214996120537d91f0bab5da66: 2024-11-23T16:15:20,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66. 2024-11-23T16:15:20,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=94 2024-11-23T16:15:20,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.HMaster(4169): Remote procedure done, pid=94 2024-11-23T16:15:20,974 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=94, resume processing ppid=93 2024-11-23T16:15:20,974 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=94, ppid=93, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0720 sec 2024-11-23T16:15:20,977 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=93, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees in 2.0770 sec 2024-11-23T16:15:21,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-23T16:15:21,059 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-23T16:15:21,059 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-23T16:15:21,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] procedure2.ProcedureExecutor(1139): Stored pid=95, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees 2024-11-23T16:15:21,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-23T16:15:21,061 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=95, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-23T16:15:21,062 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=95, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-23T16:15:21,062 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=96, ppid=95, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-23T16:15:21,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-23T16:15:21,214 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-23T16:15:21,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66. 2024-11-23T16:15:21,214 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2902): Flushing 0b4ea01214996120537d91f0bab5da66 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-23T16:15:21,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0b4ea01214996120537d91f0bab5da66, store=A 2024-11-23T16:15:21,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-23T16:15:21,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0b4ea01214996120537d91f0bab5da66, store=B 2024-11-23T16:15:21,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-23T16:15:21,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0b4ea01214996120537d91f0bab5da66, store=C 2024-11-23T16:15:21,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-23T16:15:21,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241123666d3b50debc4fa09e897cab2daa2188_0b4ea01214996120537d91f0bab5da66 is 50, key is test_row_0/A:col10/1732378519328/Put/seqid=0 2024-11-23T16:15:21,225 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742175_1351 (size=12304) 2024-11-23T16:15:21,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-23T16:15:21,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(8855): Flush requested on 0b4ea01214996120537d91f0bab5da66 2024-11-23T16:15:21,533 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66. as already flushing 2024-11-23T16:15:21,604 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:21,604 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:21,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55404 deadline: 1732378581591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:21,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55464 deadline: 1732378581587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:21,604 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:21,604 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:21,604 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:21,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55466 deadline: 1732378581598, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:21,604 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:21,604 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:21,605 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:15:21,605 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:15:21,605 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:21,605 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:21,605 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:15:21,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:21,629 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241123666d3b50debc4fa09e897cab2daa2188_0b4ea01214996120537d91f0bab5da66 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123666d3b50debc4fa09e897cab2daa2188_0b4ea01214996120537d91f0bab5da66 2024-11-23T16:15:21,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/.tmp/A/59beac914a7d47b7b53a7191f69b7973, store: [table=TestAcidGuarantees family=A region=0b4ea01214996120537d91f0bab5da66] 2024-11-23T16:15:21,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/.tmp/A/59beac914a7d47b7b53a7191f69b7973 is 175, key is test_row_0/A:col10/1732378519328/Put/seqid=0 2024-11-23T16:15:21,636 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742176_1352 (size=31105) 2024-11-23T16:15:21,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-23T16:15:21,715 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:21,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55404 deadline: 1732378581708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:21,716 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:21,716 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:21,716 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:21,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55464 deadline: 1732378581708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:21,716 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:15:21,716 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:21,716 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:21,716 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:21,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55466 deadline: 1732378581708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:21,716 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:15:21,717 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:21,717 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:21,717 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:15:21,921 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:21,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55404 deadline: 1732378581918, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:21,922 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:21,922 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:21,922 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:21,922 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:15:21,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55466 deadline: 1732378581918, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:21,922 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:21,922 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:21,922 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:15:21,935 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:21,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55464 deadline: 1732378581928, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:21,936 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:21,936 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:21,936 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:15:22,036 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=162, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/.tmp/A/59beac914a7d47b7b53a7191f69b7973 2024-11-23T16:15:22,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/.tmp/B/5d7b6e4cfd954729845e022adb58804e is 50, key is test_row_0/B:col10/1732378519328/Put/seqid=0 2024-11-23T16:15:22,049 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742177_1353 (size=12151) 2024-11-23T16:15:22,050 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=162 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/.tmp/B/5d7b6e4cfd954729845e022adb58804e 2024-11-23T16:15:22,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/.tmp/C/5af48329ede3475d86cf43ecc94f84ed is 50, key is test_row_0/C:col10/1732378519328/Put/seqid=0 2024-11-23T16:15:22,062 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742178_1354 (size=12151) 2024-11-23T16:15:22,063 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=162 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/.tmp/C/5af48329ede3475d86cf43ecc94f84ed 2024-11-23T16:15:22,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/.tmp/A/59beac914a7d47b7b53a7191f69b7973 as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/A/59beac914a7d47b7b53a7191f69b7973 2024-11-23T16:15:22,071 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/A/59beac914a7d47b7b53a7191f69b7973, entries=150, sequenceid=162, filesize=30.4 K 2024-11-23T16:15:22,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/.tmp/B/5d7b6e4cfd954729845e022adb58804e as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/B/5d7b6e4cfd954729845e022adb58804e 2024-11-23T16:15:22,079 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/B/5d7b6e4cfd954729845e022adb58804e, entries=150, sequenceid=162, filesize=11.9 K 2024-11-23T16:15:22,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/.tmp/C/5af48329ede3475d86cf43ecc94f84ed as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/C/5af48329ede3475d86cf43ecc94f84ed 2024-11-23T16:15:22,083 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/C/5af48329ede3475d86cf43ecc94f84ed, entries=150, sequenceid=162, filesize=11.9 K 2024-11-23T16:15:22,084 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 0b4ea01214996120537d91f0bab5da66 in 870ms, sequenceid=162, compaction requested=true 2024-11-23T16:15:22,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2603): Flush status journal for 0b4ea01214996120537d91f0bab5da66: 2024-11-23T16:15:22,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66. 2024-11-23T16:15:22,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=96 2024-11-23T16:15:22,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.HMaster(4169): Remote procedure done, pid=96 2024-11-23T16:15:22,087 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=96, resume processing ppid=95 2024-11-23T16:15:22,087 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=96, ppid=95, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0230 sec 2024-11-23T16:15:22,089 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=95, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees in 1.0290 sec 2024-11-23T16:15:22,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-23T16:15:22,198 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-23T16:15:22,199 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-23T16:15:22,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] procedure2.ProcedureExecutor(1139): Stored pid=97, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees 2024-11-23T16:15:22,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-23T16:15:22,201 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=97, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-23T16:15:22,202 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=97, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-23T16:15:22,202 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=98, ppid=97, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-23T16:15:22,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(8855): Flush requested on 0b4ea01214996120537d91f0bab5da66 2024-11-23T16:15:22,236 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1a7c5470b472:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732378521570, max timestamp=1732378522233Num uniques 859; 2024-11-23T16:15:22,237 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1a7c5470b472:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-23T16:15:22,237 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1a7c5470b472:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732378521570, max timestamp=1732378522233Num uniques 859; 2024-11-23T16:15:22,237 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1a7c5470b472:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-23T16:15:22,237 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1a7c5470b472:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732378521570, max timestamp=1732378522233Num uniques 859; 2024-11-23T16:15:22,237 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1a7c5470b472:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-23T16:15:22,237 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0b4ea01214996120537d91f0bab5da66 3/3 column families, dataSize=33.54 KB heapSize=86.16 KB 2024-11-23T16:15:22,237 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0b4ea01214996120537d91f0bab5da66, store=A 2024-11-23T16:15:22,238 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-23T16:15:22,238 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0b4ea01214996120537d91f0bab5da66, store=B 2024-11-23T16:15:22,238 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-23T16:15:22,238 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0b4ea01214996120537d91f0bab5da66, store=C 2024-11-23T16:15:22,238 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-23T16:15:22,250 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112367c4c16b30524e10903209650d1c819b_0b4ea01214996120537d91f0bab5da66 is 50, key is test_row_0/A:col10/1732378522235/Put/seqid=0 2024-11-23T16:15:22,279 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742179_1355 (size=17284) 2024-11-23T16:15:22,285 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:22,289 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112367c4c16b30524e10903209650d1c819b_0b4ea01214996120537d91f0bab5da66 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112367c4c16b30524e10903209650d1c819b_0b4ea01214996120537d91f0bab5da66 2024-11-23T16:15:22,290 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/.tmp/A/5cb370ec2a0e4b88a4475c3d32e9e237, store: [table=TestAcidGuarantees family=A region=0b4ea01214996120537d91f0bab5da66] 2024-11-23T16:15:22,291 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/.tmp/A/5cb370ec2a0e4b88a4475c3d32e9e237 is 175, key is test_row_0/A:col10/1732378522235/Put/seqid=0 2024-11-23T16:15:22,298 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742180_1356 (size=48389) 2024-11-23T16:15:22,299 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=184, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/.tmp/A/5cb370ec2a0e4b88a4475c3d32e9e237 2024-11-23T16:15:22,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-23T16:15:22,308 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/.tmp/B/e0ce66cdc1084169ad0e623cdb557b70 is 50, key is test_row_0/B:col10/1732378522235/Put/seqid=0 2024-11-23T16:15:22,317 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742181_1357 (size=12151) 2024-11-23T16:15:22,322 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1a7c5470b472:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732378522285, max timestamp=1732378522316Num uniques 859; 2024-11-23T16:15:22,322 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1a7c5470b472:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732378522285, max timestamp=1732378522316Num uniques 859; 2024-11-23T16:15:22,322 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1a7c5470b472:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-23T16:15:22,322 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1a7c5470b472:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-23T16:15:22,322 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=184 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/.tmp/B/e0ce66cdc1084169ad0e623cdb557b70 2024-11-23T16:15:22,322 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1a7c5470b472:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732378522285, max timestamp=1732378522316Num uniques 859; 2024-11-23T16:15:22,323 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1a7c5470b472:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-23T16:15:22,329 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/.tmp/C/5debbf60369542a6bee33065de8007f9 is 50, key is test_row_0/C:col10/1732378522235/Put/seqid=0 2024-11-23T16:15:22,337 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742182_1358 (size=12151) 2024-11-23T16:15:22,341 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=184 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/.tmp/C/5debbf60369542a6bee33065de8007f9 2024-11-23T16:15:22,346 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/.tmp/A/5cb370ec2a0e4b88a4475c3d32e9e237 as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/A/5cb370ec2a0e4b88a4475c3d32e9e237 2024-11-23T16:15:22,349 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/A/5cb370ec2a0e4b88a4475c3d32e9e237, entries=250, sequenceid=184, filesize=47.3 K 2024-11-23T16:15:22,349 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/.tmp/B/e0ce66cdc1084169ad0e623cdb557b70 as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/B/e0ce66cdc1084169ad0e623cdb557b70 2024-11-23T16:15:22,354 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-23T16:15:22,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66. 2024-11-23T16:15:22,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66. as already flushing 2024-11-23T16:15:22,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66. 2024-11-23T16:15:22,354 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 0b4ea01214996120537d91f0bab5da66, NAME => 'TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:15:22,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 0b4ea01214996120537d91f0bab5da66, NAME => 'TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:15:22,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0b4ea01214996120537d91f0bab5da66, NAME => 'TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0b4ea01214996120537d91f0bab5da66, NAME => 'TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:15:22,357 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/B/e0ce66cdc1084169ad0e623cdb557b70, entries=150, sequenceid=184, filesize=11.9 K 2024-11-23T16:15:22,358 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/.tmp/C/5debbf60369542a6bee33065de8007f9 as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/C/5debbf60369542a6bee33065de8007f9 2024-11-23T16:15:22,366 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/C/5debbf60369542a6bee33065de8007f9, entries=150, sequenceid=184, filesize=11.9 K 2024-11-23T16:15:22,367 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~86.11 KB/88176, currentSize=87.22 KB/89310 for 0b4ea01214996120537d91f0bab5da66 in 130ms, sequenceid=184, compaction requested=true 2024-11-23T16:15:22,367 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0b4ea01214996120537d91f0bab5da66: 2024-11-23T16:15:22,368 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0b4ea01214996120537d91f0bab5da66:A, priority=-2147483648, current under compaction store size is 1 2024-11-23T16:15:22,368 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T16:15:22,368 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-23T16:15:22,368 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-23T16:15:22,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(8855): Flush requested on 0b4ea01214996120537d91f0bab5da66 2024-11-23T16:15:22,369 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0b4ea01214996120537d91f0bab5da66:B, priority=-2147483648, current under compaction store size is 2 2024-11-23T16:15:22,369 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T16:15:22,369 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0b4ea01214996120537d91f0bab5da66:C, priority=-2147483648, current under compaction store size is 3 2024-11-23T16:15:22,369 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T16:15:22,369 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0b4ea01214996120537d91f0bab5da66 3/3 column families, dataSize=93.93 KB heapSize=260.77 KB 2024-11-23T16:15:22,371 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0b4ea01214996120537d91f0bab5da66, store=A 2024-11-23T16:15:22,371 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-23T16:15:22,371 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0b4ea01214996120537d91f0bab5da66, store=B 2024-11-23T16:15:22,371 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-23T16:15:22,371 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0b4ea01214996120537d91f0bab5da66, store=C 2024-11-23T16:15:22,371 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-23T16:15:22,374 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48660 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-23T16:15:22,374 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HStore(1541): 0b4ea01214996120537d91f0bab5da66/B is initiating minor compaction (all files) 2024-11-23T16:15:22,374 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0b4ea01214996120537d91f0bab5da66/B in TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66. 2024-11-23T16:15:22,374 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/B/0e7a82f77905450992d2e76e9b85cd30, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/B/00f659f3e9d44b2cb9249f39840dc25a, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/B/5d7b6e4cfd954729845e022adb58804e, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/B/e0ce66cdc1084169ad0e623cdb557b70] into tmpdir=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/.tmp, totalSize=47.5 K 2024-11-23T16:15:22,374 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 141760 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-23T16:15:22,374 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HStore(1541): 0b4ea01214996120537d91f0bab5da66/A is initiating minor compaction (all files) 2024-11-23T16:15:22,374 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0b4ea01214996120537d91f0bab5da66/A in TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66. 2024-11-23T16:15:22,374 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/A/45d9fc022931492fb4f0e113a3584846, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/A/4b4d8e58d563499c8442ce94d2dac1c8, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/A/59beac914a7d47b7b53a7191f69b7973, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/A/5cb370ec2a0e4b88a4475c3d32e9e237] into tmpdir=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/.tmp, totalSize=138.4 K 2024-11-23T16:15:22,374 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66. 2024-11-23T16:15:22,374 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66. files: [hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/A/45d9fc022931492fb4f0e113a3584846, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/A/4b4d8e58d563499c8442ce94d2dac1c8, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/A/59beac914a7d47b7b53a7191f69b7973, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/A/5cb370ec2a0e4b88a4475c3d32e9e237] 2024-11-23T16:15:22,376 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.Compactor(225): Compacting 0e7a82f77905450992d2e76e9b85cd30, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=127, earliestPutTs=1732378517333 2024-11-23T16:15:22,376 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.Compactor(225): Compacting 45d9fc022931492fb4f0e113a3584846, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=127, earliestPutTs=1732378517333 2024-11-23T16:15:22,376 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.Compactor(225): Compacting 00f659f3e9d44b2cb9249f39840dc25a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=147, earliestPutTs=1732378518616 2024-11-23T16:15:22,376 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4b4d8e58d563499c8442ce94d2dac1c8, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=147, earliestPutTs=1732378518616 2024-11-23T16:15:22,377 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.Compactor(225): Compacting 5d7b6e4cfd954729845e022adb58804e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=162, earliestPutTs=1732378519328 2024-11-23T16:15:22,377 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.Compactor(225): Compacting 59beac914a7d47b7b53a7191f69b7973, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=162, earliestPutTs=1732378519328 2024-11-23T16:15:22,377 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.Compactor(225): Compacting e0ce66cdc1084169ad0e623cdb557b70, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=184, earliestPutTs=1732378521584 2024-11-23T16:15:22,378 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5cb370ec2a0e4b88a4475c3d32e9e237, keycount=250, bloomtype=ROW, size=47.3 K, encoding=NONE, compression=NONE, seqNum=184, earliestPutTs=1732378521570 2024-11-23T16:15:22,395 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0b4ea01214996120537d91f0bab5da66#B#compaction#300 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T16:15:22,395 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/.tmp/B/1560e1e7552a449dae570a2934b12e5b is 50, key is test_row_0/B:col10/1732378522235/Put/seqid=0 2024-11-23T16:15:22,398 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=0b4ea01214996120537d91f0bab5da66] 2024-11-23T16:15:22,408 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024112393e93747f3d348338439599156e31b23_0b4ea01214996120537d91f0bab5da66 store=[table=TestAcidGuarantees family=A region=0b4ea01214996120537d91f0bab5da66] 2024-11-23T16:15:22,410 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024112393e93747f3d348338439599156e31b23_0b4ea01214996120537d91f0bab5da66, store=[table=TestAcidGuarantees family=A region=0b4ea01214996120537d91f0bab5da66] 2024-11-23T16:15:22,411 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112393e93747f3d348338439599156e31b23_0b4ea01214996120537d91f0bab5da66 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=0b4ea01214996120537d91f0bab5da66] 2024-11-23T16:15:22,418 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112307454ec926ca4cbd95680fb2d146d0ca_0b4ea01214996120537d91f0bab5da66 is 50, key is test_row_0/A:col10/1732378522368/Put/seqid=0 2024-11-23T16:15:22,420 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742183_1359 (size=12493) 2024-11-23T16:15:22,426 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/.tmp/B/1560e1e7552a449dae570a2934b12e5b as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/B/1560e1e7552a449dae570a2934b12e5b 2024-11-23T16:15:22,433 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742184_1360 (size=4469) 2024-11-23T16:15:22,436 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 0b4ea01214996120537d91f0bab5da66/B of 0b4ea01214996120537d91f0bab5da66 into 1560e1e7552a449dae570a2934b12e5b(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T16:15:22,436 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0b4ea01214996120537d91f0bab5da66: 2024-11-23T16:15:22,436 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., storeName=0b4ea01214996120537d91f0bab5da66/B, priority=12, startTime=1732378522368; duration=0sec 2024-11-23T16:15:22,436 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T16:15:22,436 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0b4ea01214996120537d91f0bab5da66:B 2024-11-23T16:15:22,436 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-23T16:15:22,437 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48660 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-23T16:15:22,438 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HStore(1541): 0b4ea01214996120537d91f0bab5da66/C is initiating minor compaction (all files) 2024-11-23T16:15:22,438 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0b4ea01214996120537d91f0bab5da66/C in TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66. 2024-11-23T16:15:22,438 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/C/0aa96d0e877843d6a90e2557993e71b5, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/C/d9293febc9d449adb396df71274727ee, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/C/5af48329ede3475d86cf43ecc94f84ed, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/C/5debbf60369542a6bee33065de8007f9] into tmpdir=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/.tmp, totalSize=47.5 K 2024-11-23T16:15:22,438 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.Compactor(225): Compacting 0aa96d0e877843d6a90e2557993e71b5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=127, earliestPutTs=1732378517333 2024-11-23T16:15:22,439 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.Compactor(225): Compacting d9293febc9d449adb396df71274727ee, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=147, earliestPutTs=1732378518616 2024-11-23T16:15:22,439 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.Compactor(225): Compacting 5af48329ede3475d86cf43ecc94f84ed, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=162, earliestPutTs=1732378519328 2024-11-23T16:15:22,439 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.Compactor(225): Compacting 5debbf60369542a6bee33065de8007f9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=184, earliestPutTs=1732378521584 2024-11-23T16:15:22,453 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742185_1361 (size=14794) 2024-11-23T16:15:22,454 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:22,457 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112307454ec926ca4cbd95680fb2d146d0ca_0b4ea01214996120537d91f0bab5da66 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112307454ec926ca4cbd95680fb2d146d0ca_0b4ea01214996120537d91f0bab5da66 2024-11-23T16:15:22,458 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/.tmp/A/b1bb64b0378541a6983cdbccd3881f7a, store: [table=TestAcidGuarantees family=A region=0b4ea01214996120537d91f0bab5da66] 2024-11-23T16:15:22,459 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/.tmp/A/b1bb64b0378541a6983cdbccd3881f7a is 175, key is test_row_0/A:col10/1732378522368/Put/seqid=0 2024-11-23T16:15:22,462 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0b4ea01214996120537d91f0bab5da66#C#compaction#303 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-23T16:15:22,463 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/.tmp/C/e9558cf58a984213b55b0088f5e7b2d9 is 50, key is test_row_0/C:col10/1732378522235/Put/seqid=0 2024-11-23T16:15:22,463 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:22,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55466 deadline: 1732378582441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:22,464 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:22,464 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:22,464 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:15:22,469 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:22,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55404 deadline: 1732378582464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:22,469 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:22,469 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:22,469 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:15:22,474 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742186_1362 (size=39749) 2024-11-23T16:15:22,475 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=215, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/.tmp/A/b1bb64b0378541a6983cdbccd3881f7a 2024-11-23T16:15:22,475 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742187_1363 (size=12493) 2024-11-23T16:15:22,481 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:22,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55464 deadline: 1732378582465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:22,482 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:22,482 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:22,482 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:15:22,482 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/.tmp/B/c56aaef115a94f7aa1a8d975fc676cc9 is 50, key is test_row_0/B:col10/1732378522368/Put/seqid=0 2024-11-23T16:15:22,489 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742188_1364 (size=12151) 2024-11-23T16:15:22,507 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-23T16:15:22,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66. 2024-11-23T16:15:22,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66. as already flushing 2024-11-23T16:15:22,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66. 2024-11-23T16:15:22,507 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 0b4ea01214996120537d91f0bab5da66, NAME => 'TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:15:22,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 0b4ea01214996120537d91f0bab5da66, NAME => 'TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:15:22,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0b4ea01214996120537d91f0bab5da66, NAME => 'TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0b4ea01214996120537d91f0bab5da66, NAME => 'TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:15:22,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-23T16:15:22,580 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:22,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55466 deadline: 1732378582568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:22,580 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:22,580 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:22,580 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:15:22,591 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:22,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55404 deadline: 1732378582577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:22,591 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:22,591 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:22,591 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:15:22,599 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:22,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55464 deadline: 1732378582588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:22,599 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:22,599 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:22,599 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:15:22,660 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-23T16:15:22,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66. 2024-11-23T16:15:22,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66. as already flushing 2024-11-23T16:15:22,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66. 2024-11-23T16:15:22,660 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 0b4ea01214996120537d91f0bab5da66, NAME => 'TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:15:22,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 0b4ea01214996120537d91f0bab5da66, NAME => 'TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:15:22,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0b4ea01214996120537d91f0bab5da66, NAME => 'TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0b4ea01214996120537d91f0bab5da66, NAME => 'TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:15:22,800 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:22,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55466 deadline: 1732378582788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:22,800 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:22,801 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:22,801 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:15:22,807 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:22,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55404 deadline: 1732378582798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:22,807 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:22,807 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:22,807 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:15:22,813 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-23T16:15:22,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66. 2024-11-23T16:15:22,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66. as already flushing 2024-11-23T16:15:22,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66. 2024-11-23T16:15:22,813 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 0b4ea01214996120537d91f0bab5da66, NAME => 'TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:15:22,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 0b4ea01214996120537d91f0bab5da66, NAME => 'TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:15:22,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0b4ea01214996120537d91f0bab5da66, NAME => 'TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0b4ea01214996120537d91f0bab5da66, NAME => 'TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:15:22,821 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:22,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55464 deadline: 1732378582807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:22,821 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:22,821 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:22,821 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:15:22,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-23T16:15:22,834 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0b4ea01214996120537d91f0bab5da66#A#compaction#301 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T16:15:22,835 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/.tmp/A/bf2c8d8c6ff2465f99ba0be092802e28 is 175, key is test_row_0/A:col10/1732378522235/Put/seqid=0 2024-11-23T16:15:22,838 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742189_1365 (size=31447) 2024-11-23T16:15:22,879 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/.tmp/C/e9558cf58a984213b55b0088f5e7b2d9 as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/C/e9558cf58a984213b55b0088f5e7b2d9 2024-11-23T16:15:22,884 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 0b4ea01214996120537d91f0bab5da66/C of 0b4ea01214996120537d91f0bab5da66 into e9558cf58a984213b55b0088f5e7b2d9(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T16:15:22,884 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0b4ea01214996120537d91f0bab5da66: 2024-11-23T16:15:22,884 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., storeName=0b4ea01214996120537d91f0bab5da66/C, priority=12, startTime=1732378522369; duration=0sec 2024-11-23T16:15:22,884 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T16:15:22,884 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0b4ea01214996120537d91f0bab5da66:C 2024-11-23T16:15:22,892 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=215 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/.tmp/B/c56aaef115a94f7aa1a8d975fc676cc9 2024-11-23T16:15:22,903 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/.tmp/C/331031eb35fc44d9a8ebeaf7c8de922c is 50, key is test_row_0/C:col10/1732378522368/Put/seqid=0 2024-11-23T16:15:22,912 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742190_1366 (size=12151) 2024-11-23T16:15:22,966 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-23T16:15:22,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66. 2024-11-23T16:15:22,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66. as already flushing 2024-11-23T16:15:22,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66. 2024-11-23T16:15:22,966 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 0b4ea01214996120537d91f0bab5da66, NAME => 'TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:15:22,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 0b4ea01214996120537d91f0bab5da66, NAME => 'TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:15:22,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0b4ea01214996120537d91f0bab5da66, NAME => 'TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0b4ea01214996120537d91f0bab5da66, NAME => 'TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:15:23,119 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-23T16:15:23,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66. 2024-11-23T16:15:23,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66. as already flushing 2024-11-23T16:15:23,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66. 2024-11-23T16:15:23,119 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 0b4ea01214996120537d91f0bab5da66, NAME => 'TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:15:23,119 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:23,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 0b4ea01214996120537d91f0bab5da66, NAME => 'TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:15:23,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55466 deadline: 1732378583108, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:23,120 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:23,120 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:23,120 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:15:23,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0b4ea01214996120537d91f0bab5da66, NAME => 'TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0b4ea01214996120537d91f0bab5da66, NAME => 'TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:15:23,129 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:23,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55404 deadline: 1732378583118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:23,130 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:23,130 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:23,130 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:15:23,139 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:23,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55464 deadline: 1732378583127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:23,140 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:23,140 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:23,140 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:15:23,244 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/.tmp/A/bf2c8d8c6ff2465f99ba0be092802e28 as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/A/bf2c8d8c6ff2465f99ba0be092802e28 2024-11-23T16:15:23,251 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 0b4ea01214996120537d91f0bab5da66/A of 0b4ea01214996120537d91f0bab5da66 into bf2c8d8c6ff2465f99ba0be092802e28(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T16:15:23,251 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0b4ea01214996120537d91f0bab5da66: 2024-11-23T16:15:23,252 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., storeName=0b4ea01214996120537d91f0bab5da66/A, priority=12, startTime=1732378522367; duration=0sec 2024-11-23T16:15:23,252 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T16:15:23,252 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0b4ea01214996120537d91f0bab5da66:A 2024-11-23T16:15:23,272 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-23T16:15:23,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66. 2024-11-23T16:15:23,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66. as already flushing 2024-11-23T16:15:23,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66. 2024-11-23T16:15:23,273 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 0b4ea01214996120537d91f0bab5da66, NAME => 'TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:15:23,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 0b4ea01214996120537d91f0bab5da66, NAME => 'TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:15:23,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0b4ea01214996120537d91f0bab5da66, NAME => 'TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0b4ea01214996120537d91f0bab5da66, NAME => 'TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:15:23,313 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=215 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/.tmp/C/331031eb35fc44d9a8ebeaf7c8de922c 2024-11-23T16:15:23,318 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/.tmp/A/b1bb64b0378541a6983cdbccd3881f7a as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/A/b1bb64b0378541a6983cdbccd3881f7a 2024-11-23T16:15:23,321 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/A/b1bb64b0378541a6983cdbccd3881f7a, entries=200, sequenceid=215, filesize=38.8 K 2024-11-23T16:15:23,322 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/.tmp/B/c56aaef115a94f7aa1a8d975fc676cc9 as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/B/c56aaef115a94f7aa1a8d975fc676cc9 2024-11-23T16:15:23,326 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/B/c56aaef115a94f7aa1a8d975fc676cc9, entries=150, sequenceid=215, filesize=11.9 K 2024-11-23T16:15:23,326 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/.tmp/C/331031eb35fc44d9a8ebeaf7c8de922c as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/C/331031eb35fc44d9a8ebeaf7c8de922c 2024-11-23T16:15:23,330 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/C/331031eb35fc44d9a8ebeaf7c8de922c, entries=150, sequenceid=215, filesize=11.9 K 2024-11-23T16:15:23,331 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~280.64 KB/287376, currentSize=87.22 KB/89310 for 0b4ea01214996120537d91f0bab5da66 in 962ms, sequenceid=215, compaction requested=false 2024-11-23T16:15:23,331 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0b4ea01214996120537d91f0bab5da66: 2024-11-23T16:15:23,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-23T16:15:23,426 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-23T16:15:23,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66. 2024-11-23T16:15:23,426 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2902): Flushing 0b4ea01214996120537d91f0bab5da66 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-23T16:15:23,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0b4ea01214996120537d91f0bab5da66, store=A 2024-11-23T16:15:23,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-23T16:15:23,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0b4ea01214996120537d91f0bab5da66, store=B 2024-11-23T16:15:23,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-23T16:15:23,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0b4ea01214996120537d91f0bab5da66, store=C 2024-11-23T16:15:23,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-23T16:15:23,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241123ce3ce2ed690244d181a2bd65f9ba5458_0b4ea01214996120537d91f0bab5da66 is 50, key is test_row_0/A:col10/1732378522462/Put/seqid=0 2024-11-23T16:15:23,438 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742191_1367 (size=12304) 2024-11-23T16:15:23,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(8855): Flush requested on 0b4ea01214996120537d91f0bab5da66 2024-11-23T16:15:23,453 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66. as already flushing 2024-11-23T16:15:23,517 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:23,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55422 deadline: 1732378583505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:23,517 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:23,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55412 deadline: 1732378583506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:23,518 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:23,518 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:23,518 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:15:23,518 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:23,518 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:23,518 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:15:23,634 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:23,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55466 deadline: 1732378583628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:23,634 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:23,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55422 deadline: 1732378583628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:23,634 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:23,634 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:23,634 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:15:23,634 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:23,635 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:23,635 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:23,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55412 deadline: 1732378583628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:23,635 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:15:23,635 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:23,635 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:23,635 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:15:23,647 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:23,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55404 deadline: 1732378583637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:23,648 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:23,648 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:23,648 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:15:23,655 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:23,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55464 deadline: 1732378583648, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:23,655 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:23,656 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:23,656 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:15:23,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:23,842 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241123ce3ce2ed690244d181a2bd65f9ba5458_0b4ea01214996120537d91f0bab5da66 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123ce3ce2ed690244d181a2bd65f9ba5458_0b4ea01214996120537d91f0bab5da66 2024-11-23T16:15:23,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/.tmp/A/0952134456d14209a6af590379db6463, store: [table=TestAcidGuarantees family=A region=0b4ea01214996120537d91f0bab5da66] 2024-11-23T16:15:23,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/.tmp/A/0952134456d14209a6af590379db6463 is 175, key is test_row_0/A:col10/1732378522462/Put/seqid=0 2024-11-23T16:15:23,847 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742192_1368 (size=31105) 2024-11-23T16:15:23,848 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:23,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55422 deadline: 1732378583838, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:23,848 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:23,848 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:23,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55412 deadline: 1732378583838, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:23,848 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:23,848 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:15:23,849 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:23,849 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:23,849 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:15:24,168 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:24,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55422 deadline: 1732378584158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:24,168 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:24,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55412 deadline: 1732378584158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:24,168 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:24,168 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:24,168 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:15:24,168 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:24,169 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:24,169 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:15:24,248 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=234, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/.tmp/A/0952134456d14209a6af590379db6463 2024-11-23T16:15:24,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/.tmp/B/f92848caab9a4196b331fa99e97d2dab is 50, key is test_row_0/B:col10/1732378522462/Put/seqid=0 2024-11-23T16:15:24,260 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742193_1369 (size=12151) 2024-11-23T16:15:24,260 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=234 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/.tmp/B/f92848caab9a4196b331fa99e97d2dab 2024-11-23T16:15:24,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/.tmp/C/1baf74d4d9ee4fd983b07941f20e80d8 is 50, key is test_row_0/C:col10/1732378522462/Put/seqid=0 2024-11-23T16:15:24,274 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742194_1370 (size=12151) 2024-11-23T16:15:24,275 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=234 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/.tmp/C/1baf74d4d9ee4fd983b07941f20e80d8 2024-11-23T16:15:24,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/.tmp/A/0952134456d14209a6af590379db6463 as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/A/0952134456d14209a6af590379db6463 2024-11-23T16:15:24,283 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/A/0952134456d14209a6af590379db6463, entries=150, sequenceid=234, filesize=30.4 K 2024-11-23T16:15:24,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/.tmp/B/f92848caab9a4196b331fa99e97d2dab as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/B/f92848caab9a4196b331fa99e97d2dab 2024-11-23T16:15:24,290 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/B/f92848caab9a4196b331fa99e97d2dab, entries=150, sequenceid=234, filesize=11.9 K 2024-11-23T16:15:24,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/.tmp/C/1baf74d4d9ee4fd983b07941f20e80d8 as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/C/1baf74d4d9ee4fd983b07941f20e80d8 2024-11-23T16:15:24,295 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/C/1baf74d4d9ee4fd983b07941f20e80d8, entries=150, sequenceid=234, filesize=11.9 K 2024-11-23T16:15:24,295 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 0b4ea01214996120537d91f0bab5da66 in 869ms, sequenceid=234, compaction requested=true 2024-11-23T16:15:24,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2603): Flush status journal for 0b4ea01214996120537d91f0bab5da66: 2024-11-23T16:15:24,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66. 2024-11-23T16:15:24,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=98 2024-11-23T16:15:24,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.HMaster(4169): Remote procedure done, pid=98 2024-11-23T16:15:24,299 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=98, resume processing ppid=97 2024-11-23T16:15:24,299 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=98, ppid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0950 sec 2024-11-23T16:15:24,300 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees in 2.1000 sec 2024-11-23T16:15:24,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-23T16:15:24,349 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-23T16:15:24,349 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-23T16:15:24,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] procedure2.ProcedureExecutor(1139): Stored pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees 2024-11-23T16:15:24,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-23T16:15:24,351 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-23T16:15:24,351 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-23T16:15:24,351 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=100, ppid=99, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-23T16:15:24,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-23T16:15:24,504 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-23T16:15:24,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66. 2024-11-23T16:15:24,504 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2902): Flushing 0b4ea01214996120537d91f0bab5da66 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-23T16:15:24,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0b4ea01214996120537d91f0bab5da66, store=A 2024-11-23T16:15:24,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-23T16:15:24,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0b4ea01214996120537d91f0bab5da66, store=B 2024-11-23T16:15:24,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-23T16:15:24,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0b4ea01214996120537d91f0bab5da66, store=C 2024-11-23T16:15:24,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-23T16:15:24,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112375a02a6cc2764cb5a1c9225b7a9767ed_0b4ea01214996120537d91f0bab5da66 is 50, key is test_row_0/A:col10/1732378523505/Put/seqid=0 2024-11-23T16:15:24,518 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742195_1371 (size=12304) 2024-11-23T16:15:24,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:24,522 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112375a02a6cc2764cb5a1c9225b7a9767ed_0b4ea01214996120537d91f0bab5da66 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112375a02a6cc2764cb5a1c9225b7a9767ed_0b4ea01214996120537d91f0bab5da66 2024-11-23T16:15:24,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/.tmp/A/8f908143c3094354995f5e2d8404bba2, store: [table=TestAcidGuarantees family=A region=0b4ea01214996120537d91f0bab5da66] 2024-11-23T16:15:24,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/.tmp/A/8f908143c3094354995f5e2d8404bba2 is 175, key is test_row_0/A:col10/1732378523505/Put/seqid=0 2024-11-23T16:15:24,527 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742196_1372 (size=31105) 2024-11-23T16:15:24,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(8855): Flush requested on 0b4ea01214996120537d91f0bab5da66 2024-11-23T16:15:24,644 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66. as already flushing 2024-11-23T16:15:24,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-23T16:15:24,693 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:24,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55466 deadline: 1732378584680, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:24,694 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:24,694 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:24,694 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:15:24,694 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:24,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55404 deadline: 1732378584686, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:24,694 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:24,694 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:24,694 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:15:24,699 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:24,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55464 deadline: 1732378584691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:24,700 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:24,700 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:24,700 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:15:24,708 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:24,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55422 deadline: 1732378584692, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:24,708 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:24,708 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:24,708 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:24,708 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:15:24,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55412 deadline: 1732378584693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:24,709 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:24,709 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:24,709 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:15:24,809 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:24,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55466 deadline: 1732378584797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:24,810 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:24,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55404 deadline: 1732378584797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:24,810 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:24,810 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:24,810 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:15:24,810 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:24,810 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:24,810 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:15:24,815 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:24,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55464 deadline: 1732378584808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:24,816 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:24,816 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:24,816 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:15:24,830 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:24,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55422 deadline: 1732378584818, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:24,830 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:24,830 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:24,830 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:24,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55412 deadline: 1732378584818, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:24,830 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:15:24,831 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:24,831 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:24,831 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:15:24,928 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=250, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/.tmp/A/8f908143c3094354995f5e2d8404bba2 2024-11-23T16:15:24,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/.tmp/B/b0391390cad647918a967ef86fb57475 is 50, key is test_row_0/B:col10/1732378523505/Put/seqid=0 2024-11-23T16:15:24,939 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742197_1373 (size=12151) 2024-11-23T16:15:24,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-23T16:15:25,028 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:25,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55466 deadline: 1732378585018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:25,029 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:25,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55404 deadline: 1732378585018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:25,029 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:25,029 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:25,029 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:15:25,029 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:25,029 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:25,029 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:15:25,034 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:25,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55464 deadline: 1732378585028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:25,034 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:25,035 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:25,035 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:15:25,049 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:25,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55422 deadline: 1732378585037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:25,050 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:25,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55412 deadline: 1732378585037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:25,050 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:25,050 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:25,050 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:15:25,050 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:25,050 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:25,050 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:15:25,340 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=250 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/.tmp/B/b0391390cad647918a967ef86fb57475 2024-11-23T16:15:25,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/.tmp/C/1094bffb05654d9487b9c5d5168fcdfc is 50, key is test_row_0/C:col10/1732378523505/Put/seqid=0 2024-11-23T16:15:25,348 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:25,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55466 deadline: 1732378585338, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:25,348 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:25,349 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:25,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55404 deadline: 1732378585338, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:25,349 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:25,349 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:15:25,349 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:25,349 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:25,349 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:15:25,351 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742198_1374 (size=12151) 2024-11-23T16:15:25,355 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:25,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55464 deadline: 1732378585348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:25,355 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:25,355 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:25,356 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:15:25,370 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:25,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55422 deadline: 1732378585357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:25,370 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:25,370 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:25,370 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:25,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55412 deadline: 1732378585357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:25,370 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:15:25,371 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:25,371 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:25,371 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:15:25,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-23T16:15:25,752 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=250 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/.tmp/C/1094bffb05654d9487b9c5d5168fcdfc 2024-11-23T16:15:25,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/.tmp/A/8f908143c3094354995f5e2d8404bba2 as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/A/8f908143c3094354995f5e2d8404bba2 2024-11-23T16:15:25,761 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/A/8f908143c3094354995f5e2d8404bba2, entries=150, sequenceid=250, filesize=30.4 K 2024-11-23T16:15:25,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/.tmp/B/b0391390cad647918a967ef86fb57475 as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/B/b0391390cad647918a967ef86fb57475 2024-11-23T16:15:25,768 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/B/b0391390cad647918a967ef86fb57475, entries=150, sequenceid=250, filesize=11.9 K 2024-11-23T16:15:25,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/.tmp/C/1094bffb05654d9487b9c5d5168fcdfc as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/C/1094bffb05654d9487b9c5d5168fcdfc 2024-11-23T16:15:25,772 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/C/1094bffb05654d9487b9c5d5168fcdfc, entries=150, sequenceid=250, filesize=11.9 K 2024-11-23T16:15:25,773 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 0b4ea01214996120537d91f0bab5da66 in 1269ms, sequenceid=250, compaction requested=true 2024-11-23T16:15:25,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2603): Flush status journal for 0b4ea01214996120537d91f0bab5da66: 2024-11-23T16:15:25,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66. 2024-11-23T16:15:25,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=100 2024-11-23T16:15:25,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.HMaster(4169): Remote procedure done, pid=100 2024-11-23T16:15:25,777 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=100, resume processing ppid=99 2024-11-23T16:15:25,777 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=100, ppid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4240 sec 2024-11-23T16:15:25,779 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees in 1.4290 sec 2024-11-23T16:15:25,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(8855): Flush requested on 0b4ea01214996120537d91f0bab5da66 2024-11-23T16:15:25,865 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0b4ea01214996120537d91f0bab5da66 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-23T16:15:25,866 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0b4ea01214996120537d91f0bab5da66, store=A 2024-11-23T16:15:25,866 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-23T16:15:25,866 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0b4ea01214996120537d91f0bab5da66, store=B 2024-11-23T16:15:25,866 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-23T16:15:25,866 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0b4ea01214996120537d91f0bab5da66, store=C 2024-11-23T16:15:25,866 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-23T16:15:25,875 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411236e7c8e9ccc044179b96b3a549ff9aec5_0b4ea01214996120537d91f0bab5da66 is 50, key is test_row_0/A:col10/1732378524674/Put/seqid=0 2024-11-23T16:15:25,878 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742199_1375 (size=14994) 2024-11-23T16:15:25,881 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:25,884 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411236e7c8e9ccc044179b96b3a549ff9aec5_0b4ea01214996120537d91f0bab5da66 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411236e7c8e9ccc044179b96b3a549ff9aec5_0b4ea01214996120537d91f0bab5da66 2024-11-23T16:15:25,885 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/.tmp/A/3b1b572cdf7d4235a2e9170ced416c17, store: [table=TestAcidGuarantees family=A region=0b4ea01214996120537d91f0bab5da66] 2024-11-23T16:15:25,885 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/.tmp/A/3b1b572cdf7d4235a2e9170ced416c17 is 175, key is test_row_0/A:col10/1732378524674/Put/seqid=0 2024-11-23T16:15:25,889 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742200_1376 (size=39949) 2024-11-23T16:15:25,890 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=267, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/.tmp/A/3b1b572cdf7d4235a2e9170ced416c17 2024-11-23T16:15:25,897 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/.tmp/B/9404e71d2dad4911bb3a0a1e22f54cbd is 50, key is test_row_0/B:col10/1732378524674/Put/seqid=0 2024-11-23T16:15:25,900 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742201_1377 (size=12301) 2024-11-23T16:15:25,915 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:25,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55404 deadline: 1732378585896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:25,915 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:25,915 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:25,916 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:25,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55464 deadline: 1732378585897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:25,916 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:15:25,916 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:25,916 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:25,916 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:25,916 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:15:25,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55466 deadline: 1732378585897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:25,916 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:25,916 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:25,916 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:25,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55422 deadline: 1732378585899, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:25,916 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:15:25,916 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:25,917 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:25,917 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:15:25,928 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:25,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55412 deadline: 1732378585915, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:25,928 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:25,928 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:25,928 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:15:26,032 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:26,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55404 deadline: 1732378586018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:26,033 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:26,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55464 deadline: 1732378586018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:26,033 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:26,033 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:26,033 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:15:26,033 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:26,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55466 deadline: 1732378586018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:26,033 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:26,033 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:26,033 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:15:26,033 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:26,033 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:26,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55422 deadline: 1732378586018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:26,033 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:26,033 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:15:26,034 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:26,034 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:26,034 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:15:26,050 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:26,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55412 deadline: 1732378586038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:26,050 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:26,050 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:26,050 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:15:26,251 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:26,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55404 deadline: 1732378586237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:26,252 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:26,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55464 deadline: 1732378586237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:26,252 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:26,252 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:26,252 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:15:26,252 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:26,252 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:26,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55466 deadline: 1732378586238, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:26,252 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:26,252 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:15:26,252 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:26,252 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:26,252 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:26,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55422 deadline: 1732378586238, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:26,252 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:15:26,253 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:26,253 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:26,253 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:15:26,270 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:26,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55412 deadline: 1732378586258, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:26,270 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:26,270 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:26,270 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:15:26,301 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=267 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/.tmp/B/9404e71d2dad4911bb3a0a1e22f54cbd 2024-11-23T16:15:26,307 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/.tmp/C/8d6b85562863486e9b61551a77d38369 is 50, key is test_row_0/C:col10/1732378524674/Put/seqid=0 2024-11-23T16:15:26,312 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742202_1378 (size=12301) 2024-11-23T16:15:26,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-23T16:15:26,498 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-23T16:15:26,499 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-23T16:15:26,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] procedure2.ProcedureExecutor(1139): Stored pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees 2024-11-23T16:15:26,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-23T16:15:26,500 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-23T16:15:26,501 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-23T16:15:26,501 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=102, ppid=101, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-23T16:15:26,568 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:26,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55404 deadline: 1732378586557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:26,568 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:26,568 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:26,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55464 deadline: 1732378586558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:26,568 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:26,568 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:15:26,569 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:26,569 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:26,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55466 deadline: 1732378586558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:26,569 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:26,569 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:15:26,569 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:26,569 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:26,569 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:26,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55422 deadline: 1732378586558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:26,569 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:15:26,569 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:26,569 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:26,569 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:15:26,584 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:26,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55412 deadline: 1732378586578, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:26,585 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:26,585 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:26,585 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:15:26,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-23T16:15:26,653 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-23T16:15:26,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66. 2024-11-23T16:15:26,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66. as already flushing 2024-11-23T16:15:26,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66. 2024-11-23T16:15:26,653 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 0b4ea01214996120537d91f0bab5da66, NAME => 'TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:15:26,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 0b4ea01214996120537d91f0bab5da66, NAME => 'TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:15:26,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0b4ea01214996120537d91f0bab5da66, NAME => 'TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0b4ea01214996120537d91f0bab5da66, NAME => 'TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:15:26,713 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=267 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/.tmp/C/8d6b85562863486e9b61551a77d38369 2024-11-23T16:15:26,719 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/.tmp/A/3b1b572cdf7d4235a2e9170ced416c17 as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/A/3b1b572cdf7d4235a2e9170ced416c17 2024-11-23T16:15:26,727 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/A/3b1b572cdf7d4235a2e9170ced416c17, entries=200, sequenceid=267, filesize=39.0 K 2024-11-23T16:15:26,728 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/.tmp/B/9404e71d2dad4911bb3a0a1e22f54cbd as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/B/9404e71d2dad4911bb3a0a1e22f54cbd 2024-11-23T16:15:26,732 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/B/9404e71d2dad4911bb3a0a1e22f54cbd, entries=150, sequenceid=267, filesize=12.0 K 2024-11-23T16:15:26,737 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/.tmp/C/8d6b85562863486e9b61551a77d38369 as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/C/8d6b85562863486e9b61551a77d38369 2024-11-23T16:15:26,742 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/C/8d6b85562863486e9b61551a77d38369, entries=150, sequenceid=267, filesize=12.0 K 2024-11-23T16:15:26,743 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 0b4ea01214996120537d91f0bab5da66 in 878ms, sequenceid=267, compaction requested=true 2024-11-23T16:15:26,743 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0b4ea01214996120537d91f0bab5da66: 2024-11-23T16:15:26,743 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0b4ea01214996120537d91f0bab5da66:A, priority=-2147483648, current under compaction store size is 1 2024-11-23T16:15:26,743 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T16:15:26,743 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0b4ea01214996120537d91f0bab5da66:B, priority=-2147483648, current under compaction store size is 2 2024-11-23T16:15:26,744 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T16:15:26,743 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-23T16:15:26,744 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-23T16:15:26,744 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0b4ea01214996120537d91f0bab5da66:C, priority=-2147483648, current under compaction store size is 3 2024-11-23T16:15:26,744 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T16:15:26,745 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 61247 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-23T16:15:26,745 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HStore(1541): 0b4ea01214996120537d91f0bab5da66/B is initiating minor compaction (all files) 2024-11-23T16:15:26,745 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0b4ea01214996120537d91f0bab5da66/B in TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66. 2024-11-23T16:15:26,745 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/B/1560e1e7552a449dae570a2934b12e5b, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/B/c56aaef115a94f7aa1a8d975fc676cc9, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/B/f92848caab9a4196b331fa99e97d2dab, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/B/b0391390cad647918a967ef86fb57475, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/B/9404e71d2dad4911bb3a0a1e22f54cbd] into tmpdir=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/.tmp, totalSize=59.8 K 2024-11-23T16:15:26,745 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 173355 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-23T16:15:26,746 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HStore(1541): 0b4ea01214996120537d91f0bab5da66/A is initiating minor compaction (all files) 2024-11-23T16:15:26,746 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0b4ea01214996120537d91f0bab5da66/A in TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66. 2024-11-23T16:15:26,746 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.Compactor(225): Compacting 1560e1e7552a449dae570a2934b12e5b, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=184, earliestPutTs=1732378521584 2024-11-23T16:15:26,746 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/A/bf2c8d8c6ff2465f99ba0be092802e28, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/A/b1bb64b0378541a6983cdbccd3881f7a, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/A/0952134456d14209a6af590379db6463, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/A/8f908143c3094354995f5e2d8404bba2, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/A/3b1b572cdf7d4235a2e9170ced416c17] into tmpdir=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/.tmp, totalSize=169.3 K 2024-11-23T16:15:26,746 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=11 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66. 2024-11-23T16:15:26,746 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66. files: [hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/A/bf2c8d8c6ff2465f99ba0be092802e28, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/A/b1bb64b0378541a6983cdbccd3881f7a, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/A/0952134456d14209a6af590379db6463, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/A/8f908143c3094354995f5e2d8404bba2, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/A/3b1b572cdf7d4235a2e9170ced416c17] 2024-11-23T16:15:26,746 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.Compactor(225): Compacting c56aaef115a94f7aa1a8d975fc676cc9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=215, earliestPutTs=1732378522343 2024-11-23T16:15:26,746 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.Compactor(225): Compacting bf2c8d8c6ff2465f99ba0be092802e28, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=184, earliestPutTs=1732378521584 2024-11-23T16:15:26,746 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.Compactor(225): Compacting f92848caab9a4196b331fa99e97d2dab, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=234, earliestPutTs=1732378522439 2024-11-23T16:15:26,746 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.Compactor(225): Compacting b1bb64b0378541a6983cdbccd3881f7a, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=215, earliestPutTs=1732378522340 2024-11-23T16:15:26,746 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.Compactor(225): Compacting b0391390cad647918a967ef86fb57475, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=250, earliestPutTs=1732378523472 2024-11-23T16:15:26,747 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0952134456d14209a6af590379db6463, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=234, earliestPutTs=1732378522439 2024-11-23T16:15:26,747 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.Compactor(225): Compacting 9404e71d2dad4911bb3a0a1e22f54cbd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=267, earliestPutTs=1732378524674 2024-11-23T16:15:26,747 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8f908143c3094354995f5e2d8404bba2, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=250, earliestPutTs=1732378523472 2024-11-23T16:15:26,747 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3b1b572cdf7d4235a2e9170ced416c17, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=267, earliestPutTs=1732378524674 2024-11-23T16:15:26,757 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=0b4ea01214996120537d91f0bab5da66] 2024-11-23T16:15:26,759 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241123a855994d72da4b8fbfead83b0100b804_0b4ea01214996120537d91f0bab5da66 store=[table=TestAcidGuarantees family=A region=0b4ea01214996120537d91f0bab5da66] 2024-11-23T16:15:26,760 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0b4ea01214996120537d91f0bab5da66#B#compaction#315 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-23T16:15:26,761 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/.tmp/B/4ff6b8de730b416cbde479cdfa30b570 is 50, key is test_row_0/B:col10/1732378524674/Put/seqid=0 2024-11-23T16:15:26,764 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241123a855994d72da4b8fbfead83b0100b804_0b4ea01214996120537d91f0bab5da66, store=[table=TestAcidGuarantees family=A region=0b4ea01214996120537d91f0bab5da66] 2024-11-23T16:15:26,764 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241123a855994d72da4b8fbfead83b0100b804_0b4ea01214996120537d91f0bab5da66 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=0b4ea01214996120537d91f0bab5da66] 2024-11-23T16:15:26,770 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742203_1379 (size=12813) 2024-11-23T16:15:26,775 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/.tmp/B/4ff6b8de730b416cbde479cdfa30b570 as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/B/4ff6b8de730b416cbde479cdfa30b570 2024-11-23T16:15:26,779 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 0b4ea01214996120537d91f0bab5da66/B of 0b4ea01214996120537d91f0bab5da66 into 4ff6b8de730b416cbde479cdfa30b570(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T16:15:26,779 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0b4ea01214996120537d91f0bab5da66: 2024-11-23T16:15:26,779 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., storeName=0b4ea01214996120537d91f0bab5da66/B, priority=11, startTime=1732378526743; duration=0sec 2024-11-23T16:15:26,779 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T16:15:26,779 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0b4ea01214996120537d91f0bab5da66:B 2024-11-23T16:15:26,779 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-23T16:15:26,781 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 61247 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-23T16:15:26,781 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HStore(1541): 0b4ea01214996120537d91f0bab5da66/C is initiating minor compaction (all files) 2024-11-23T16:15:26,781 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0b4ea01214996120537d91f0bab5da66/C in TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66. 2024-11-23T16:15:26,781 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/C/e9558cf58a984213b55b0088f5e7b2d9, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/C/331031eb35fc44d9a8ebeaf7c8de922c, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/C/1baf74d4d9ee4fd983b07941f20e80d8, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/C/1094bffb05654d9487b9c5d5168fcdfc, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/C/8d6b85562863486e9b61551a77d38369] into tmpdir=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/.tmp, totalSize=59.8 K 2024-11-23T16:15:26,782 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.Compactor(225): Compacting e9558cf58a984213b55b0088f5e7b2d9, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=184, earliestPutTs=1732378521584 2024-11-23T16:15:26,782 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742204_1380 (size=4469) 2024-11-23T16:15:26,782 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.Compactor(225): Compacting 331031eb35fc44d9a8ebeaf7c8de922c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=215, earliestPutTs=1732378522343 2024-11-23T16:15:26,783 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.Compactor(225): Compacting 1baf74d4d9ee4fd983b07941f20e80d8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=234, earliestPutTs=1732378522439 2024-11-23T16:15:26,783 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0b4ea01214996120537d91f0bab5da66#A#compaction#316 average throughput is 0.94 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T16:15:26,783 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.Compactor(225): Compacting 1094bffb05654d9487b9c5d5168fcdfc, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=250, earliestPutTs=1732378523472 2024-11-23T16:15:26,783 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.Compactor(225): Compacting 8d6b85562863486e9b61551a77d38369, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=267, earliestPutTs=1732378524674 2024-11-23T16:15:26,785 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/.tmp/A/8fe6b959d04d44c1acdf9770855dbca4 is 175, key is test_row_0/A:col10/1732378524674/Put/seqid=0 2024-11-23T16:15:26,789 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742205_1381 (size=31767) 2024-11-23T16:15:26,793 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0b4ea01214996120537d91f0bab5da66#C#compaction#317 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T16:15:26,793 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/.tmp/C/cf8bbe7d16204bdaa6a41a337821c588 is 50, key is test_row_0/C:col10/1732378524674/Put/seqid=0 2024-11-23T16:15:26,796 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742206_1382 (size=12813) 2024-11-23T16:15:26,806 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-23T16:15:26,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66. 2024-11-23T16:15:26,806 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2902): Flushing 0b4ea01214996120537d91f0bab5da66 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-23T16:15:26,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0b4ea01214996120537d91f0bab5da66, store=A 2024-11-23T16:15:26,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-23T16:15:26,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0b4ea01214996120537d91f0bab5da66, store=B 2024-11-23T16:15:26,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-23T16:15:26,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0b4ea01214996120537d91f0bab5da66, store=C 2024-11-23T16:15:26,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-23T16:15:26,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112376fb74abada1481eaac8f227110edb8e_0b4ea01214996120537d91f0bab5da66 is 50, key is test_row_0/A:col10/1732378525896/Put/seqid=0 2024-11-23T16:15:26,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-23T16:15:26,818 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742207_1383 (size=12454) 2024-11-23T16:15:27,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(8855): Flush requested on 0b4ea01214996120537d91f0bab5da66 2024-11-23T16:15:27,085 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66. as already flushing 2024-11-23T16:15:27,121 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:27,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55464 deadline: 1732378587107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:27,122 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:27,122 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:27,122 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:15:27,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-23T16:15:27,134 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:27,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55466 deadline: 1732378587119, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:27,135 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:27,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55422 deadline: 1732378587120, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:27,135 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:27,135 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:27,135 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:15:27,135 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:27,135 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:27,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55404 deadline: 1732378587121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:27,135 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:27,135 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:15:27,135 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:27,135 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:27,135 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:27,135 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:15:27,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55412 deadline: 1732378587121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:27,136 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:27,136 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:27,136 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:15:27,194 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/.tmp/A/8fe6b959d04d44c1acdf9770855dbca4 as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/A/8fe6b959d04d44c1acdf9770855dbca4 2024-11-23T16:15:27,200 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 0b4ea01214996120537d91f0bab5da66/A of 0b4ea01214996120537d91f0bab5da66 into 8fe6b959d04d44c1acdf9770855dbca4(size=31.0 K), total size for store is 31.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T16:15:27,200 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0b4ea01214996120537d91f0bab5da66: 2024-11-23T16:15:27,200 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., storeName=0b4ea01214996120537d91f0bab5da66/A, priority=11, startTime=1732378526743; duration=0sec 2024-11-23T16:15:27,200 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T16:15:27,200 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0b4ea01214996120537d91f0bab5da66:A 2024-11-23T16:15:27,202 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/.tmp/C/cf8bbe7d16204bdaa6a41a337821c588 as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/C/cf8bbe7d16204bdaa6a41a337821c588 2024-11-23T16:15:27,206 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 0b4ea01214996120537d91f0bab5da66/C of 0b4ea01214996120537d91f0bab5da66 into cf8bbe7d16204bdaa6a41a337821c588(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T16:15:27,206 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0b4ea01214996120537d91f0bab5da66: 2024-11-23T16:15:27,206 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., storeName=0b4ea01214996120537d91f0bab5da66/C, priority=11, startTime=1732378526744; duration=0sec 2024-11-23T16:15:27,206 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T16:15:27,206 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0b4ea01214996120537d91f0bab5da66:C 2024-11-23T16:15:27,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:27,222 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112376fb74abada1481eaac8f227110edb8e_0b4ea01214996120537d91f0bab5da66 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112376fb74abada1481eaac8f227110edb8e_0b4ea01214996120537d91f0bab5da66 2024-11-23T16:15:27,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/.tmp/A/fe634562ca1644929eaa7ba7d31d198b, store: [table=TestAcidGuarantees family=A region=0b4ea01214996120537d91f0bab5da66] 2024-11-23T16:15:27,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/.tmp/A/fe634562ca1644929eaa7ba7d31d198b is 175, key is test_row_0/A:col10/1732378525896/Put/seqid=0 2024-11-23T16:15:27,230 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742208_1384 (size=31255) 2024-11-23T16:15:27,235 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:27,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55464 deadline: 1732378587227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:27,236 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:27,236 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:27,236 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:15:27,249 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:27,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55466 deadline: 1732378587238, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:27,249 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:27,249 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:27,249 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:27,249 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:15:27,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55422 deadline: 1732378587238, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:27,250 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:27,250 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:27,250 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:27,250 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:15:27,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55404 deadline: 1732378587238, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:27,250 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:27,250 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:27,250 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:27,250 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:15:27,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55412 deadline: 1732378587238, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:27,250 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:27,251 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:27,251 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:15:27,453 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:27,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55464 deadline: 1732378587448, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:27,453 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:27,454 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:27,454 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:15:27,468 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:27,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55466 deadline: 1732378587458, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:27,469 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:27,469 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:27,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55422 deadline: 1732378587458, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:27,469 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:27,469 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:15:27,469 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:27,469 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:27,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55404 deadline: 1732378587458, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:27,469 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:27,470 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:15:27,470 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:27,470 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:27,470 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:27,470 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:15:27,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55412 deadline: 1732378587458, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:27,470 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:27,470 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:27,470 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:15:27,631 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=283, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/.tmp/A/fe634562ca1644929eaa7ba7d31d198b 2024-11-23T16:15:27,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-23T16:15:27,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/.tmp/B/610eb10921f54452b7dcc45c2fad22e1 is 50, key is test_row_0/B:col10/1732378525896/Put/seqid=0 2024-11-23T16:15:27,646 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742209_1385 (size=12301) 2024-11-23T16:15:27,646 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=283 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/.tmp/B/610eb10921f54452b7dcc45c2fad22e1 2024-11-23T16:15:27,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/.tmp/C/85b4a4b5f67f4bceb5f9297497435fba is 50, key is test_row_0/C:col10/1732378525896/Put/seqid=0 2024-11-23T16:15:27,659 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742210_1386 (size=12301) 2024-11-23T16:15:27,768 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:27,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55464 deadline: 1732378587758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:27,768 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:27,768 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:27,768 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:15:27,787 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:27,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55466 deadline: 1732378587778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:27,787 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:27,787 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:27,787 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:27,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55422 deadline: 1732378587778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:27,787 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:15:27,787 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:27,787 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:27,787 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:15:27,787 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:27,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55404 deadline: 1732378587778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:27,788 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:27,788 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:27,788 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:27,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55412 deadline: 1732378587778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:27,788 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:15:27,788 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:27,788 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:27,788 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:15:28,057 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=283 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/.tmp/C/85b4a4b5f67f4bceb5f9297497435fba 2024-11-23T16:15:28,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/.tmp/A/fe634562ca1644929eaa7ba7d31d198b as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/A/fe634562ca1644929eaa7ba7d31d198b 2024-11-23T16:15:28,065 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/A/fe634562ca1644929eaa7ba7d31d198b, entries=150, sequenceid=283, filesize=30.5 K 2024-11-23T16:15:28,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/.tmp/B/610eb10921f54452b7dcc45c2fad22e1 as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/B/610eb10921f54452b7dcc45c2fad22e1 2024-11-23T16:15:28,069 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/B/610eb10921f54452b7dcc45c2fad22e1, entries=150, sequenceid=283, filesize=12.0 K 2024-11-23T16:15:28,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/.tmp/C/85b4a4b5f67f4bceb5f9297497435fba as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/C/85b4a4b5f67f4bceb5f9297497435fba 2024-11-23T16:15:28,073 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/C/85b4a4b5f67f4bceb5f9297497435fba, entries=150, sequenceid=283, filesize=12.0 K 2024-11-23T16:15:28,075 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=100.63 KB/103050 for 0b4ea01214996120537d91f0bab5da66 in 1269ms, sequenceid=283, compaction requested=false 2024-11-23T16:15:28,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2603): Flush status journal for 0b4ea01214996120537d91f0bab5da66: 2024-11-23T16:15:28,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66. 2024-11-23T16:15:28,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=102 2024-11-23T16:15:28,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.HMaster(4169): Remote procedure done, pid=102 2024-11-23T16:15:28,078 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=102, resume processing ppid=101 2024-11-23T16:15:28,078 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=102, ppid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5750 sec 2024-11-23T16:15:28,079 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees in 1.5790 sec 2024-11-23T16:15:28,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(8855): Flush requested on 0b4ea01214996120537d91f0bab5da66 2024-11-23T16:15:28,280 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0b4ea01214996120537d91f0bab5da66 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-23T16:15:28,280 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0b4ea01214996120537d91f0bab5da66, store=A 2024-11-23T16:15:28,280 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-23T16:15:28,280 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0b4ea01214996120537d91f0bab5da66, store=B 2024-11-23T16:15:28,280 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-23T16:15:28,280 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0b4ea01214996120537d91f0bab5da66, store=C 2024-11-23T16:15:28,280 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-23T16:15:28,287 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112351c0a7908f6240e69b1bf96880325bd1_0b4ea01214996120537d91f0bab5da66 is 50, key is test_row_0/A:col10/1732378527118/Put/seqid=0 2024-11-23T16:15:28,293 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742211_1387 (size=14994) 2024-11-23T16:15:28,322 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:28,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55464 deadline: 1732378588309, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:28,323 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:28,323 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:28,323 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:28,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55466 deadline: 1732378588311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:28,323 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:15:28,323 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:28,323 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:28,323 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:15:28,333 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:28,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55422 deadline: 1732378588322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:28,334 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:28,334 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:28,334 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:28,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55404 deadline: 1732378588323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:28,334 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:15:28,334 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:28,334 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:28,334 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:28,334 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:15:28,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55412 deadline: 1732378588323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:28,334 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:28,334 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:28,334 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:15:28,433 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:28,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55464 deadline: 1732378588428, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:28,433 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:28,433 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:28,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55466 deadline: 1732378588428, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:28,433 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:28,434 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:15:28,434 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:28,434 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:28,434 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:15:28,446 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:28,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55422 deadline: 1732378588438, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:28,446 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:28,446 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:28,446 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:28,446 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:15:28,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55404 deadline: 1732378588438, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:28,447 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:28,447 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:28,447 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:28,447 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:15:28,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55412 deadline: 1732378588438, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:28,447 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:28,447 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:28,447 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:15:28,646 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:28,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55464 deadline: 1732378588638, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:28,646 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:28,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55466 deadline: 1732378588638, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:28,646 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:28,646 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:28,646 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:15:28,647 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:28,647 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:28,647 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:15:28,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-23T16:15:28,648 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-23T16:15:28,648 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-23T16:15:28,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] procedure2.ProcedureExecutor(1139): Stored pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees 2024-11-23T16:15:28,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-23T16:15:28,650 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-23T16:15:28,651 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-23T16:15:28,651 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=104, ppid=103, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-23T16:15:28,660 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:28,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55422 deadline: 1732378588658, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:28,660 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:28,660 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:28,660 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:28,660 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:15:28,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55404 deadline: 1732378588658, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:28,661 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:28,661 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:28,661 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:28,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55412 deadline: 1732378588658, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:28,661 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:15:28,661 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:28,661 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:28,661 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:15:28,694 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:28,698 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112351c0a7908f6240e69b1bf96880325bd1_0b4ea01214996120537d91f0bab5da66 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112351c0a7908f6240e69b1bf96880325bd1_0b4ea01214996120537d91f0bab5da66 2024-11-23T16:15:28,699 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/.tmp/A/0f783016cec5405aa300d7d3e870d897, store: [table=TestAcidGuarantees family=A region=0b4ea01214996120537d91f0bab5da66] 2024-11-23T16:15:28,700 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/.tmp/A/0f783016cec5405aa300d7d3e870d897 is 175, key is test_row_0/A:col10/1732378527118/Put/seqid=0 2024-11-23T16:15:28,704 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742212_1388 (size=39949) 2024-11-23T16:15:28,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-23T16:15:28,802 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-23T16:15:28,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66. 2024-11-23T16:15:28,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66. as already flushing 2024-11-23T16:15:28,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66. 2024-11-23T16:15:28,803 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 0b4ea01214996120537d91f0bab5da66, NAME => 'TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:15:28,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 0b4ea01214996120537d91f0bab5da66, NAME => 'TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:15:28,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0b4ea01214996120537d91f0bab5da66, NAME => 'TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0b4ea01214996120537d91f0bab5da66, NAME => 'TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:15:28,949 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:28,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55464 deadline: 1732378588948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:28,949 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:28,949 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:28,949 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:15:28,955 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-23T16:15:28,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66. 2024-11-23T16:15:28,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66. as already flushing 2024-11-23T16:15:28,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66. 2024-11-23T16:15:28,956 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 0b4ea01214996120537d91f0bab5da66, NAME => 'TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:15:28,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 0b4ea01214996120537d91f0bab5da66, NAME => 'TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:15:28,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0b4ea01214996120537d91f0bab5da66, NAME => 'TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0b4ea01214996120537d91f0bab5da66, NAME => 'TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:15:28,961 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:28,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55466 deadline: 1732378588958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:28,961 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:28,961 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:28,961 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:15:28,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-23T16:15:28,972 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:28,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55422 deadline: 1732378588968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:28,972 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:28,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55404 deadline: 1732378588968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:28,972 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:28,972 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:28,972 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:15:28,973 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:28,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55412 deadline: 1732378588968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:28,973 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:28,973 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:28,973 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:15:28,973 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:28,973 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:28,973 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:15:29,105 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=304, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/.tmp/A/0f783016cec5405aa300d7d3e870d897 2024-11-23T16:15:29,109 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-23T16:15:29,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66. 2024-11-23T16:15:29,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66. as already flushing 2024-11-23T16:15:29,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66. 2024-11-23T16:15:29,109 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 0b4ea01214996120537d91f0bab5da66, NAME => 'TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:15:29,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 0b4ea01214996120537d91f0bab5da66, NAME => 'TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:15:29,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0b4ea01214996120537d91f0bab5da66, NAME => 'TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0b4ea01214996120537d91f0bab5da66, NAME => 'TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:15:29,113 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/.tmp/B/cd2729ddd6c846e8aeb90dd926d2790e is 50, key is test_row_0/B:col10/1732378527118/Put/seqid=0 2024-11-23T16:15:29,117 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742213_1389 (size=12301) 2024-11-23T16:15:29,262 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-23T16:15:29,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66. 2024-11-23T16:15:29,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66. as already flushing 2024-11-23T16:15:29,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66. 2024-11-23T16:15:29,263 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 0b4ea01214996120537d91f0bab5da66, NAME => 'TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:15:29,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 0b4ea01214996120537d91f0bab5da66, NAME => 'TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:15:29,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0b4ea01214996120537d91f0bab5da66, NAME => 'TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0b4ea01214996120537d91f0bab5da66, NAME => 'TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:15:29,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-23T16:15:29,415 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-23T16:15:29,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66. 2024-11-23T16:15:29,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66. as already flushing 2024-11-23T16:15:29,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66. 2024-11-23T16:15:29,416 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 0b4ea01214996120537d91f0bab5da66, NAME => 'TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:15:29,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 0b4ea01214996120537d91f0bab5da66, NAME => 'TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:15:29,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0b4ea01214996120537d91f0bab5da66, NAME => 'TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0b4ea01214996120537d91f0bab5da66, NAME => 'TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:15:29,464 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:29,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55464 deadline: 1732378589458, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:29,464 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:29,464 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:29,464 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:15:29,474 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:29,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55466 deadline: 1732378589467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:29,475 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:29,475 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:29,475 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:15:29,486 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:29,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55422 deadline: 1732378589477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:29,486 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:29,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55404 deadline: 1732378589478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:29,486 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:29,486 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:29,486 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:15:29,486 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:29,486 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:29,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55412 deadline: 1732378589478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:29,486 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:29,486 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:15:29,486 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:29,487 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:29,487 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:15:29,517 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=304 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/.tmp/B/cd2729ddd6c846e8aeb90dd926d2790e 2024-11-23T16:15:29,524 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/.tmp/C/257345c6b0e04480850d4a9c3bd0b6e2 is 50, key is test_row_0/C:col10/1732378527118/Put/seqid=0 2024-11-23T16:15:29,527 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742214_1390 (size=12301) 2024-11-23T16:15:29,568 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-23T16:15:29,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66. 2024-11-23T16:15:29,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66. as already flushing 2024-11-23T16:15:29,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66. 2024-11-23T16:15:29,569 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 0b4ea01214996120537d91f0bab5da66, NAME => 'TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:15:29,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 0b4ea01214996120537d91f0bab5da66, NAME => 'TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:15:29,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0b4ea01214996120537d91f0bab5da66, NAME => 'TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0b4ea01214996120537d91f0bab5da66, NAME => 'TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:15:29,722 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-23T16:15:29,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66. 2024-11-23T16:15:29,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66. as already flushing 2024-11-23T16:15:29,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66. 2024-11-23T16:15:29,722 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 0b4ea01214996120537d91f0bab5da66, NAME => 'TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:15:29,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 0b4ea01214996120537d91f0bab5da66, NAME => 'TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:15:29,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0b4ea01214996120537d91f0bab5da66, NAME => 'TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0b4ea01214996120537d91f0bab5da66, NAME => 'TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:15:29,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-23T16:15:29,876 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-23T16:15:29,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66. 2024-11-23T16:15:29,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66. as already flushing 2024-11-23T16:15:29,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66. 2024-11-23T16:15:29,877 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 0b4ea01214996120537d91f0bab5da66, NAME => 'TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:15:29,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 0b4ea01214996120537d91f0bab5da66, NAME => 'TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:15:29,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0b4ea01214996120537d91f0bab5da66, NAME => 'TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0b4ea01214996120537d91f0bab5da66, NAME => 'TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:15:29,929 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=304 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/.tmp/C/257345c6b0e04480850d4a9c3bd0b6e2 2024-11-23T16:15:29,937 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/.tmp/A/0f783016cec5405aa300d7d3e870d897 as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/A/0f783016cec5405aa300d7d3e870d897 2024-11-23T16:15:29,942 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/A/0f783016cec5405aa300d7d3e870d897, entries=200, sequenceid=304, filesize=39.0 K 2024-11-23T16:15:29,943 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/.tmp/B/cd2729ddd6c846e8aeb90dd926d2790e as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/B/cd2729ddd6c846e8aeb90dd926d2790e 2024-11-23T16:15:29,948 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/B/cd2729ddd6c846e8aeb90dd926d2790e, entries=150, sequenceid=304, filesize=12.0 K 2024-11-23T16:15:29,949 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/.tmp/C/257345c6b0e04480850d4a9c3bd0b6e2 as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/C/257345c6b0e04480850d4a9c3bd0b6e2 2024-11-23T16:15:29,956 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/C/257345c6b0e04480850d4a9c3bd0b6e2, entries=150, sequenceid=304, filesize=12.0 K 2024-11-23T16:15:29,957 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=73.80 KB/75570 for 0b4ea01214996120537d91f0bab5da66 in 1677ms, sequenceid=304, compaction requested=true 2024-11-23T16:15:29,957 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0b4ea01214996120537d91f0bab5da66: 2024-11-23T16:15:29,957 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0b4ea01214996120537d91f0bab5da66:A, priority=-2147483648, current under compaction store size is 1 2024-11-23T16:15:29,957 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T16:15:29,957 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T16:15:29,957 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T16:15:29,958 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0b4ea01214996120537d91f0bab5da66:B, priority=-2147483648, current under compaction store size is 2 2024-11-23T16:15:29,958 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T16:15:29,958 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0b4ea01214996120537d91f0bab5da66:C, priority=-2147483648, current under compaction store size is 3 2024-11-23T16:15:29,958 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T16:15:29,961 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37415 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T16:15:29,961 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HStore(1541): 0b4ea01214996120537d91f0bab5da66/B is initiating minor compaction (all files) 2024-11-23T16:15:29,961 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0b4ea01214996120537d91f0bab5da66/B in TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66. 2024-11-23T16:15:29,961 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/B/4ff6b8de730b416cbde479cdfa30b570, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/B/610eb10921f54452b7dcc45c2fad22e1, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/B/cd2729ddd6c846e8aeb90dd926d2790e] into tmpdir=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/.tmp, totalSize=36.5 K 2024-11-23T16:15:29,961 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102971 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T16:15:29,961 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HStore(1541): 0b4ea01214996120537d91f0bab5da66/A is initiating minor compaction (all files) 2024-11-23T16:15:29,961 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0b4ea01214996120537d91f0bab5da66/A in TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66. 2024-11-23T16:15:29,961 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.Compactor(225): Compacting 4ff6b8de730b416cbde479cdfa30b570, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=267, earliestPutTs=1732378524674 2024-11-23T16:15:29,961 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/A/8fe6b959d04d44c1acdf9770855dbca4, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/A/fe634562ca1644929eaa7ba7d31d198b, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/A/0f783016cec5405aa300d7d3e870d897] into tmpdir=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/.tmp, totalSize=100.6 K 2024-11-23T16:15:29,962 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66. 2024-11-23T16:15:29,962 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66. files: [hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/A/8fe6b959d04d44c1acdf9770855dbca4, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/A/fe634562ca1644929eaa7ba7d31d198b, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/A/0f783016cec5405aa300d7d3e870d897] 2024-11-23T16:15:29,962 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.Compactor(225): Compacting 610eb10921f54452b7dcc45c2fad22e1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=283, earliestPutTs=1732378525880 2024-11-23T16:15:29,962 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8fe6b959d04d44c1acdf9770855dbca4, keycount=150, bloomtype=ROW, size=31.0 K, encoding=NONE, compression=NONE, seqNum=267, earliestPutTs=1732378524674 2024-11-23T16:15:29,962 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.Compactor(225): Compacting cd2729ddd6c846e8aeb90dd926d2790e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=304, earliestPutTs=1732378527101 2024-11-23T16:15:29,963 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.Compactor(225): Compacting fe634562ca1644929eaa7ba7d31d198b, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=283, earliestPutTs=1732378525880 2024-11-23T16:15:29,963 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0f783016cec5405aa300d7d3e870d897, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=304, earliestPutTs=1732378527101 2024-11-23T16:15:29,985 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=0b4ea01214996120537d91f0bab5da66] 2024-11-23T16:15:29,989 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411232f3475c7271949958b5f165a516ef27b_0b4ea01214996120537d91f0bab5da66 store=[table=TestAcidGuarantees family=A region=0b4ea01214996120537d91f0bab5da66] 2024-11-23T16:15:29,990 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0b4ea01214996120537d91f0bab5da66#B#compaction#325 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-23T16:15:29,991 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/.tmp/B/25608670fa694a57997894b6648a844e is 50, key is test_row_0/B:col10/1732378527118/Put/seqid=0 2024-11-23T16:15:29,991 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411232f3475c7271949958b5f165a516ef27b_0b4ea01214996120537d91f0bab5da66, store=[table=TestAcidGuarantees family=A region=0b4ea01214996120537d91f0bab5da66] 2024-11-23T16:15:29,992 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411232f3475c7271949958b5f165a516ef27b_0b4ea01214996120537d91f0bab5da66 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=0b4ea01214996120537d91f0bab5da66] 2024-11-23T16:15:30,022 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742216_1392 (size=4469) 2024-11-23T16:15:30,027 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0b4ea01214996120537d91f0bab5da66#A#compaction#324 average throughput is 0.58 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T16:15:30,028 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/.tmp/A/5d610660216d4613800ca624da9ebefa is 175, key is test_row_0/A:col10/1732378527118/Put/seqid=0 2024-11-23T16:15:30,032 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-23T16:15:30,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66. 2024-11-23T16:15:30,033 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2902): Flushing 0b4ea01214996120537d91f0bab5da66 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-23T16:15:30,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0b4ea01214996120537d91f0bab5da66, store=A 2024-11-23T16:15:30,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-23T16:15:30,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0b4ea01214996120537d91f0bab5da66, store=B 2024-11-23T16:15:30,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-23T16:15:30,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0b4ea01214996120537d91f0bab5da66, store=C 2024-11-23T16:15:30,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-23T16:15:30,056 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742215_1391 (size=12915) 2024-11-23T16:15:30,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411235805391473964cbcaade71fd7c21caf1_0b4ea01214996120537d91f0bab5da66 is 50, key is test_row_0/A:col10/1732378528305/Put/seqid=0 2024-11-23T16:15:30,070 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742217_1393 (size=31869) 2024-11-23T16:15:30,070 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/.tmp/B/25608670fa694a57997894b6648a844e as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/B/25608670fa694a57997894b6648a844e 2024-11-23T16:15:30,077 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0b4ea01214996120537d91f0bab5da66/B of 0b4ea01214996120537d91f0bab5da66 into 25608670fa694a57997894b6648a844e(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T16:15:30,077 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0b4ea01214996120537d91f0bab5da66: 2024-11-23T16:15:30,077 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., storeName=0b4ea01214996120537d91f0bab5da66/B, priority=13, startTime=1732378529957; duration=0sec 2024-11-23T16:15:30,077 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T16:15:30,077 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0b4ea01214996120537d91f0bab5da66:B 2024-11-23T16:15:30,077 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T16:15:30,079 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/.tmp/A/5d610660216d4613800ca624da9ebefa as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/A/5d610660216d4613800ca624da9ebefa 2024-11-23T16:15:30,079 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37415 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T16:15:30,079 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HStore(1541): 0b4ea01214996120537d91f0bab5da66/C is initiating minor compaction (all files) 2024-11-23T16:15:30,079 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0b4ea01214996120537d91f0bab5da66/C in TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66. 2024-11-23T16:15:30,079 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/C/cf8bbe7d16204bdaa6a41a337821c588, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/C/85b4a4b5f67f4bceb5f9297497435fba, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/C/257345c6b0e04480850d4a9c3bd0b6e2] into tmpdir=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/.tmp, totalSize=36.5 K 2024-11-23T16:15:30,080 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.Compactor(225): Compacting cf8bbe7d16204bdaa6a41a337821c588, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=267, earliestPutTs=1732378524674 2024-11-23T16:15:30,080 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.Compactor(225): Compacting 85b4a4b5f67f4bceb5f9297497435fba, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=283, earliestPutTs=1732378525880 2024-11-23T16:15:30,081 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.Compactor(225): Compacting 257345c6b0e04480850d4a9c3bd0b6e2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=304, earliestPutTs=1732378527101 2024-11-23T16:15:30,085 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0b4ea01214996120537d91f0bab5da66/A of 0b4ea01214996120537d91f0bab5da66 into 5d610660216d4613800ca624da9ebefa(size=31.1 K), total size for store is 31.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T16:15:30,085 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0b4ea01214996120537d91f0bab5da66: 2024-11-23T16:15:30,085 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., storeName=0b4ea01214996120537d91f0bab5da66/A, priority=13, startTime=1732378529957; duration=0sec 2024-11-23T16:15:30,085 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T16:15:30,085 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0b4ea01214996120537d91f0bab5da66:A 2024-11-23T16:15:30,103 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0b4ea01214996120537d91f0bab5da66#C#compaction#327 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T16:15:30,103 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/.tmp/C/0d79d4e151d541dd9cd4c4d81844bf99 is 50, key is test_row_0/C:col10/1732378527118/Put/seqid=0 2024-11-23T16:15:30,118 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742218_1394 (size=12454) 2024-11-23T16:15:30,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:30,125 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411235805391473964cbcaade71fd7c21caf1_0b4ea01214996120537d91f0bab5da66 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411235805391473964cbcaade71fd7c21caf1_0b4ea01214996120537d91f0bab5da66 2024-11-23T16:15:30,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/.tmp/A/240a8e340d65488fb925a32c1a2dfd8d, store: [table=TestAcidGuarantees family=A region=0b4ea01214996120537d91f0bab5da66] 2024-11-23T16:15:30,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/.tmp/A/240a8e340d65488fb925a32c1a2dfd8d is 175, key is test_row_0/A:col10/1732378528305/Put/seqid=0 2024-11-23T16:15:30,138 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742219_1395 (size=12915) 2024-11-23T16:15:30,153 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/.tmp/C/0d79d4e151d541dd9cd4c4d81844bf99 as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/C/0d79d4e151d541dd9cd4c4d81844bf99 2024-11-23T16:15:30,160 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0b4ea01214996120537d91f0bab5da66/C of 0b4ea01214996120537d91f0bab5da66 into 0d79d4e151d541dd9cd4c4d81844bf99(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T16:15:30,160 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0b4ea01214996120537d91f0bab5da66: 2024-11-23T16:15:30,160 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., storeName=0b4ea01214996120537d91f0bab5da66/C, priority=13, startTime=1732378529958; duration=0sec 2024-11-23T16:15:30,161 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T16:15:30,161 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0b4ea01214996120537d91f0bab5da66:C 2024-11-23T16:15:30,175 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742220_1396 (size=31255) 2024-11-23T16:15:30,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(8855): Flush requested on 0b4ea01214996120537d91f0bab5da66 2024-11-23T16:15:30,483 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66. as already flushing 2024-11-23T16:15:30,526 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:30,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55422 deadline: 1732378590514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:30,527 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:30,527 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:30,527 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:15:30,527 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:30,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55404 deadline: 1732378590518, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:30,528 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:30,528 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:30,528 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:15:30,532 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:30,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55466 deadline: 1732378590519, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:30,532 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:30,532 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:30,532 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:15:30,540 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:30,540 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:30,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55412 deadline: 1732378590527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:30,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55464 deadline: 1732378590527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:30,541 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:30,541 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:30,541 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:15:30,541 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:30,541 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:30,541 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:15:30,576 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=318, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/.tmp/A/240a8e340d65488fb925a32c1a2dfd8d 2024-11-23T16:15:30,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/.tmp/B/dbb79f5eb7c14870bef5b1ab5d57ce3d is 50, key is test_row_0/B:col10/1732378528305/Put/seqid=0 2024-11-23T16:15:30,587 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742221_1397 (size=12301) 2024-11-23T16:15:30,648 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:30,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55422 deadline: 1732378590637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:30,649 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:30,649 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:30,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55404 deadline: 1732378590638, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:30,649 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:30,649 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:15:30,649 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:30,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55466 deadline: 1732378590638, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:30,649 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:30,649 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:30,649 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:30,649 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:30,649 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:15:30,649 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:15:30,655 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:30,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55464 deadline: 1732378590648, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:30,655 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:30,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55412 deadline: 1732378590648, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:30,655 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:30,656 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:30,656 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:15:30,656 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:30,656 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:30,656 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:15:30,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-23T16:15:30,865 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:30,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55422 deadline: 1732378590858, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:30,865 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:30,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55466 deadline: 1732378590858, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:30,865 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:30,865 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:30,865 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:15:30,865 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:30,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55404 deadline: 1732378590858, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:30,865 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:30,865 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:30,866 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:15:30,866 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:30,866 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:30,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55464 deadline: 1732378590858, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:30,866 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:30,866 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:15:30,866 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:30,866 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:30,866 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:15:30,878 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:30,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55412 deadline: 1732378590867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:30,878 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:30,878 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:30,878 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:15:30,987 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=318 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/.tmp/B/dbb79f5eb7c14870bef5b1ab5d57ce3d 2024-11-23T16:15:30,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/.tmp/C/9ab3f9205d1245489a754d369f2a3700 is 50, key is test_row_0/C:col10/1732378528305/Put/seqid=0 2024-11-23T16:15:30,999 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742222_1398 (size=12301) 2024-11-23T16:15:31,170 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:31,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55464 deadline: 1732378591167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:31,171 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:31,171 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:31,171 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:15:31,184 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:31,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55422 deadline: 1732378591178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:31,184 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:31,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55404 deadline: 1732378591178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:31,184 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:31,185 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:31,185 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:15:31,185 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:31,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55466 deadline: 1732378591178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:31,185 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:31,185 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:31,185 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:15:31,185 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:31,185 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:31,185 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:15:31,197 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:31,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55412 deadline: 1732378591188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:31,198 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:31,198 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:31,198 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:15:31,399 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=318 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/.tmp/C/9ab3f9205d1245489a754d369f2a3700 2024-11-23T16:15:31,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/.tmp/A/240a8e340d65488fb925a32c1a2dfd8d as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/A/240a8e340d65488fb925a32c1a2dfd8d 2024-11-23T16:15:31,407 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/A/240a8e340d65488fb925a32c1a2dfd8d, entries=150, sequenceid=318, filesize=30.5 K 2024-11-23T16:15:31,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/.tmp/B/dbb79f5eb7c14870bef5b1ab5d57ce3d as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/B/dbb79f5eb7c14870bef5b1ab5d57ce3d 2024-11-23T16:15:31,411 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/B/dbb79f5eb7c14870bef5b1ab5d57ce3d, entries=150, sequenceid=318, filesize=12.0 K 2024-11-23T16:15:31,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/.tmp/C/9ab3f9205d1245489a754d369f2a3700 as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/C/9ab3f9205d1245489a754d369f2a3700 2024-11-23T16:15:31,422 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/C/9ab3f9205d1245489a754d369f2a3700, entries=150, sequenceid=318, filesize=12.0 K 2024-11-23T16:15:31,423 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=107.34 KB/109920 for 0b4ea01214996120537d91f0bab5da66 in 1391ms, sequenceid=318, compaction requested=false 2024-11-23T16:15:31,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2603): Flush status journal for 0b4ea01214996120537d91f0bab5da66: 2024-11-23T16:15:31,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66. 2024-11-23T16:15:31,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=104 2024-11-23T16:15:31,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.HMaster(4169): Remote procedure done, pid=104 2024-11-23T16:15:31,427 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=104, resume processing ppid=103 2024-11-23T16:15:31,427 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=104, ppid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.7740 sec 2024-11-23T16:15:31,430 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees in 2.7800 sec 2024-11-23T16:15:31,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(8855): Flush requested on 0b4ea01214996120537d91f0bab5da66 2024-11-23T16:15:31,683 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0b4ea01214996120537d91f0bab5da66 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-23T16:15:31,683 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0b4ea01214996120537d91f0bab5da66, store=A 2024-11-23T16:15:31,683 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-23T16:15:31,683 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0b4ea01214996120537d91f0bab5da66, store=B 2024-11-23T16:15:31,684 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-23T16:15:31,684 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0b4ea01214996120537d91f0bab5da66, store=C 2024-11-23T16:15:31,684 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-23T16:15:31,692 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411232f772b236d29426fa9f4e3049561c9fd_0b4ea01214996120537d91f0bab5da66 is 50, key is test_row_0/A:col10/1732378530518/Put/seqid=0 2024-11-23T16:15:31,698 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742223_1399 (size=14994) 2024-11-23T16:15:31,719 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:31,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55466 deadline: 1732378591707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:31,719 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:31,719 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:31,719 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:31,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55412 deadline: 1732378591707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:31,719 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:15:31,719 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:31,720 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:31,720 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:15:31,720 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:31,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55422 deadline: 1732378591712, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:31,720 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:31,720 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:31,720 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:15:31,725 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:31,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55404 deadline: 1732378591718, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:31,725 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:31,725 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:31,725 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:15:31,730 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:31,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55464 deadline: 1732378591719, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:31,731 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:31,731 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:31,731 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:15:31,838 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:31,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55466 deadline: 1732378591827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:31,839 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:31,839 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:31,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55422 deadline: 1732378591828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:31,839 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:31,839 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:15:31,839 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:31,839 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:31,839 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:31,839 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:15:31,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55404 deadline: 1732378591828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:31,840 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:31,840 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:31,840 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:15:31,847 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:31,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55464 deadline: 1732378591838, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:31,848 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:31,848 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:31,848 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:15:32,056 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:32,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55466 deadline: 1732378592047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:32,057 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:32,057 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:32,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55422 deadline: 1732378592048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:32,057 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:32,057 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:15:32,057 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:32,057 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:32,057 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:32,057 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:15:32,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55404 deadline: 1732378592048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:32,057 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:32,057 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:32,057 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:15:32,063 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:32,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55464 deadline: 1732378592058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:32,064 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:32,064 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:32,064 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:15:32,098 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:32,101 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411232f772b236d29426fa9f4e3049561c9fd_0b4ea01214996120537d91f0bab5da66 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411232f772b236d29426fa9f4e3049561c9fd_0b4ea01214996120537d91f0bab5da66 2024-11-23T16:15:32,102 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/.tmp/A/f7561d02a2be4e0db1f4ac729f372ad3, store: [table=TestAcidGuarantees family=A region=0b4ea01214996120537d91f0bab5da66] 2024-11-23T16:15:32,103 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/.tmp/A/f7561d02a2be4e0db1f4ac729f372ad3 is 175, key is test_row_0/A:col10/1732378530518/Put/seqid=0 2024-11-23T16:15:32,106 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742224_1400 (size=39949) 2024-11-23T16:15:32,380 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:32,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55466 deadline: 1732378592368, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:32,380 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:32,380 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:32,380 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:32,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55422 deadline: 1732378592368, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:32,380 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:15:32,381 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:32,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55404 deadline: 1732378592369, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:32,381 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:32,381 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:32,381 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:15:32,381 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:32,381 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:32,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55464 deadline: 1732378592369, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:32,381 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:32,381 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:15:32,381 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:32,381 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:32,381 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:15:32,507 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=341, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/.tmp/A/f7561d02a2be4e0db1f4ac729f372ad3 2024-11-23T16:15:32,514 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/.tmp/B/7a03aac3067e4f678940ab196181f5e3 is 50, key is test_row_0/B:col10/1732378530518/Put/seqid=0 2024-11-23T16:15:32,517 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742225_1401 (size=12301) 2024-11-23T16:15:32,594 INFO [Thread-1475 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1475. 2024-11-23T16:15:32,594 DEBUG [Thread-1475 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-23T16:15:32,594 DEBUG [Thread-1475 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T16:15:32,595 DEBUG [Thread-1475 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T16:15:32,595 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-23T16:15:32,595 INFO [Thread-1477 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1477. 2024-11-23T16:15:32,595 DEBUG [Thread-1477 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-23T16:15:32,595 DEBUG [Thread-1477 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T16:15:32,595 DEBUG [Thread-1477 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T16:15:32,595 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-23T16:15:32,596 INFO [Thread-1473 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1473. 2024-11-23T16:15:32,596 DEBUG [Thread-1473 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-23T16:15:32,596 DEBUG [Thread-1473 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T16:15:32,596 DEBUG [Thread-1473 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T16:15:32,596 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-23T16:15:32,605 INFO [Thread-1479 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1479. 2024-11-23T16:15:32,606 DEBUG [Thread-1479 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-23T16:15:32,606 DEBUG [Thread-1479 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T16:15:32,606 INFO [Thread-1471 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1471. 2024-11-23T16:15:32,606 DEBUG [Thread-1479 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T16:15:32,606 DEBUG [Thread-1471 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-23T16:15:32,606 DEBUG [Thread-1471 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T16:15:32,606 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-23T16:15:32,606 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-23T16:15:32,606 DEBUG [Thread-1471 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T16:15:32,728 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:32,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55412 deadline: 1732378592728, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:32,729 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 1a7c5470b472,44715,1732378420375 for 'test_row_0' in 0b4ea01214996120537d91f0bab5da66 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2201 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-23T16:15:32,729 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:32,729 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:32,729 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:15:32,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-23T16:15:32,808 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-23T16:15:32,888 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:32,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55466 deadline: 1732378592888, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:32,888 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:32,888 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:32,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55404 deadline: 1732378592888, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:32,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55422 deadline: 1732378592888, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:32,888 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:32,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55464 deadline: 1732378592888, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:32,888 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:32,888 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:32,888 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:15:32,888 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:32,888 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:32,888 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:32,888 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:32,888 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:15:32,888 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:15:32,889 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:32,889 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:32,889 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:15:32,918 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=341 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/.tmp/B/7a03aac3067e4f678940ab196181f5e3 2024-11-23T16:15:32,923 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/.tmp/C/1dd00b256cf443d2978113ab2031fff7 is 50, key is test_row_0/C:col10/1732378530518/Put/seqid=0 2024-11-23T16:15:32,926 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742226_1402 (size=12301) 2024-11-23T16:15:33,327 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=341 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/.tmp/C/1dd00b256cf443d2978113ab2031fff7 2024-11-23T16:15:33,331 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/.tmp/A/f7561d02a2be4e0db1f4ac729f372ad3 as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/A/f7561d02a2be4e0db1f4ac729f372ad3 2024-11-23T16:15:33,334 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/A/f7561d02a2be4e0db1f4ac729f372ad3, entries=200, sequenceid=341, filesize=39.0 K 2024-11-23T16:15:33,334 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/.tmp/B/7a03aac3067e4f678940ab196181f5e3 as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/B/7a03aac3067e4f678940ab196181f5e3 2024-11-23T16:15:33,337 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/B/7a03aac3067e4f678940ab196181f5e3, entries=150, sequenceid=341, filesize=12.0 K 2024-11-23T16:15:33,338 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/.tmp/C/1dd00b256cf443d2978113ab2031fff7 as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/C/1dd00b256cf443d2978113ab2031fff7 2024-11-23T16:15:33,340 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/C/1dd00b256cf443d2978113ab2031fff7, entries=150, sequenceid=341, filesize=12.0 K 2024-11-23T16:15:33,341 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 0b4ea01214996120537d91f0bab5da66 in 1658ms, sequenceid=341, compaction requested=true 2024-11-23T16:15:33,341 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0b4ea01214996120537d91f0bab5da66: 2024-11-23T16:15:33,341 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0b4ea01214996120537d91f0bab5da66:A, priority=-2147483648, current under compaction store size is 1 2024-11-23T16:15:33,341 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T16:15:33,341 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0b4ea01214996120537d91f0bab5da66:B, priority=-2147483648, current under compaction store size is 2 2024-11-23T16:15:33,341 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T16:15:33,341 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T16:15:33,341 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0b4ea01214996120537d91f0bab5da66:C, priority=-2147483648, current under compaction store size is 3 2024-11-23T16:15:33,341 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T16:15:33,341 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T16:15:33,342 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103073 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T16:15:33,342 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HStore(1541): 0b4ea01214996120537d91f0bab5da66/A is initiating minor compaction (all files) 2024-11-23T16:15:33,342 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37517 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T16:15:33,342 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0b4ea01214996120537d91f0bab5da66/A in TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66. 2024-11-23T16:15:33,342 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HStore(1541): 0b4ea01214996120537d91f0bab5da66/B is initiating minor compaction (all files) 2024-11-23T16:15:33,342 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0b4ea01214996120537d91f0bab5da66/B in TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66. 2024-11-23T16:15:33,342 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/A/5d610660216d4613800ca624da9ebefa, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/A/240a8e340d65488fb925a32c1a2dfd8d, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/A/f7561d02a2be4e0db1f4ac729f372ad3] into tmpdir=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/.tmp, totalSize=100.7 K 2024-11-23T16:15:33,342 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66. 2024-11-23T16:15:33,342 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/B/25608670fa694a57997894b6648a844e, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/B/dbb79f5eb7c14870bef5b1ab5d57ce3d, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/B/7a03aac3067e4f678940ab196181f5e3] into tmpdir=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/.tmp, totalSize=36.6 K 2024-11-23T16:15:33,342 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66. files: [hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/A/5d610660216d4613800ca624da9ebefa, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/A/240a8e340d65488fb925a32c1a2dfd8d, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/A/f7561d02a2be4e0db1f4ac729f372ad3] 2024-11-23T16:15:33,342 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5d610660216d4613800ca624da9ebefa, keycount=150, bloomtype=ROW, size=31.1 K, encoding=NONE, compression=NONE, seqNum=304, earliestPutTs=1732378527101 2024-11-23T16:15:33,343 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.Compactor(225): Compacting 25608670fa694a57997894b6648a844e, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=304, earliestPutTs=1732378527101 2024-11-23T16:15:33,343 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.Compactor(225): Compacting 240a8e340d65488fb925a32c1a2dfd8d, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=318, earliestPutTs=1732378528301 2024-11-23T16:15:33,343 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.Compactor(225): Compacting dbb79f5eb7c14870bef5b1ab5d57ce3d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=318, earliestPutTs=1732378528301 2024-11-23T16:15:33,343 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.Compactor(225): Compacting f7561d02a2be4e0db1f4ac729f372ad3, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=341, earliestPutTs=1732378530518 2024-11-23T16:15:33,343 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.Compactor(225): Compacting 7a03aac3067e4f678940ab196181f5e3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=341, earliestPutTs=1732378530518 2024-11-23T16:15:33,350 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0b4ea01214996120537d91f0bab5da66#B#compaction#333 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T16:15:33,350 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/.tmp/B/7f84e325eaa44fc2a37509f40d0c9cf9 is 50, key is test_row_0/B:col10/1732378530518/Put/seqid=0 2024-11-23T16:15:33,352 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=0b4ea01214996120537d91f0bab5da66] 2024-11-23T16:15:33,356 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241123e7096d7fd45549e592d96eb5562d4eee_0b4ea01214996120537d91f0bab5da66 store=[table=TestAcidGuarantees family=A region=0b4ea01214996120537d91f0bab5da66] 2024-11-23T16:15:33,357 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742227_1403 (size=13017) 2024-11-23T16:15:33,375 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241123e7096d7fd45549e592d96eb5562d4eee_0b4ea01214996120537d91f0bab5da66, store=[table=TestAcidGuarantees family=A region=0b4ea01214996120537d91f0bab5da66] 2024-11-23T16:15:33,376 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241123e7096d7fd45549e592d96eb5562d4eee_0b4ea01214996120537d91f0bab5da66 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=0b4ea01214996120537d91f0bab5da66] 2024-11-23T16:15:33,381 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742228_1404 (size=4469) 2024-11-23T16:15:33,761 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/.tmp/B/7f84e325eaa44fc2a37509f40d0c9cf9 as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/B/7f84e325eaa44fc2a37509f40d0c9cf9 2024-11-23T16:15:33,765 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0b4ea01214996120537d91f0bab5da66/B of 0b4ea01214996120537d91f0bab5da66 into 7f84e325eaa44fc2a37509f40d0c9cf9(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T16:15:33,765 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0b4ea01214996120537d91f0bab5da66: 2024-11-23T16:15:33,765 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., storeName=0b4ea01214996120537d91f0bab5da66/B, priority=13, startTime=1732378533341; duration=0sec 2024-11-23T16:15:33,765 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T16:15:33,765 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0b4ea01214996120537d91f0bab5da66:B 2024-11-23T16:15:33,765 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T16:15:33,766 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37517 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T16:15:33,766 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HStore(1541): 0b4ea01214996120537d91f0bab5da66/C is initiating minor compaction (all files) 2024-11-23T16:15:33,766 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0b4ea01214996120537d91f0bab5da66/C in TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66. 2024-11-23T16:15:33,766 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/C/0d79d4e151d541dd9cd4c4d81844bf99, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/C/9ab3f9205d1245489a754d369f2a3700, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/C/1dd00b256cf443d2978113ab2031fff7] into tmpdir=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/.tmp, totalSize=36.6 K 2024-11-23T16:15:33,766 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.Compactor(225): Compacting 0d79d4e151d541dd9cd4c4d81844bf99, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=304, earliestPutTs=1732378527101 2024-11-23T16:15:33,766 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.Compactor(225): Compacting 9ab3f9205d1245489a754d369f2a3700, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=318, earliestPutTs=1732378528301 2024-11-23T16:15:33,767 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.Compactor(225): Compacting 1dd00b256cf443d2978113ab2031fff7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=341, earliestPutTs=1732378530518 2024-11-23T16:15:33,775 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0b4ea01214996120537d91f0bab5da66#C#compaction#335 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-23T16:15:33,776 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/.tmp/C/8469622c02294afe8d2b7b1191b84c21 is 50, key is test_row_0/C:col10/1732378530518/Put/seqid=0 2024-11-23T16:15:33,779 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742229_1405 (size=13017) 2024-11-23T16:15:33,782 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0b4ea01214996120537d91f0bab5da66#A#compaction#334 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T16:15:33,783 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/.tmp/A/f0f556b04fb841acaec352677c1e4c24 is 175, key is test_row_0/A:col10/1732378530518/Put/seqid=0 2024-11-23T16:15:33,786 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742230_1406 (size=31971) 2024-11-23T16:15:33,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(8855): Flush requested on 0b4ea01214996120537d91f0bab5da66 2024-11-23T16:15:33,899 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0b4ea01214996120537d91f0bab5da66 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-23T16:15:33,900 INFO [Thread-1468 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1468. 2024-11-23T16:15:33,900 DEBUG [Thread-1468 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-23T16:15:33,900 INFO [Thread-1460 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1460. 2024-11-23T16:15:33,900 DEBUG [Thread-1468 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T16:15:33,900 INFO [Thread-1466 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1466. 2024-11-23T16:15:33,900 DEBUG [Thread-1460 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-23T16:15:33,900 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0b4ea01214996120537d91f0bab5da66, store=A 2024-11-23T16:15:33,900 DEBUG [Thread-1460 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T16:15:33,900 DEBUG [Thread-1466 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-23T16:15:33,900 DEBUG [Thread-1466 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T16:15:33,900 DEBUG [Thread-1468 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T16:15:33,900 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-23T16:15:33,900 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-23T16:15:33,900 DEBUG [Thread-1466 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T16:15:33,900 DEBUG [Thread-1460 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T16:15:33,900 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0b4ea01214996120537d91f0bab5da66, store=B 2024-11-23T16:15:33,900 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-23T16:15:33,900 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0b4ea01214996120537d91f0bab5da66, store=C 2024-11-23T16:15:33,900 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-23T16:15:33,900 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-23T16:15:33,900 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-23T16:15:33,901 INFO [Thread-1462 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1462. 2024-11-23T16:15:33,901 DEBUG [Thread-1462 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-23T16:15:33,901 DEBUG [Thread-1462 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T16:15:33,901 DEBUG [Thread-1462 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T16:15:33,901 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-23T16:15:33,906 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411231b8ee75d80e04384b35b2b04c0e1ac89_0b4ea01214996120537d91f0bab5da66 is 50, key is test_row_0/A:col10/1732378533898/Put/seqid=0 2024-11-23T16:15:33,909 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742231_1407 (size=12454) 2024-11-23T16:15:34,183 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/.tmp/C/8469622c02294afe8d2b7b1191b84c21 as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/C/8469622c02294afe8d2b7b1191b84c21 2024-11-23T16:15:34,187 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0b4ea01214996120537d91f0bab5da66/C of 0b4ea01214996120537d91f0bab5da66 into 8469622c02294afe8d2b7b1191b84c21(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T16:15:34,187 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0b4ea01214996120537d91f0bab5da66: 2024-11-23T16:15:34,187 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., storeName=0b4ea01214996120537d91f0bab5da66/C, priority=13, startTime=1732378533341; duration=0sec 2024-11-23T16:15:34,187 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T16:15:34,187 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0b4ea01214996120537d91f0bab5da66:C 2024-11-23T16:15:34,189 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/.tmp/A/f0f556b04fb841acaec352677c1e4c24 as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/A/f0f556b04fb841acaec352677c1e4c24 2024-11-23T16:15:34,192 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0b4ea01214996120537d91f0bab5da66/A of 0b4ea01214996120537d91f0bab5da66 into f0f556b04fb841acaec352677c1e4c24(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T16:15:34,193 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0b4ea01214996120537d91f0bab5da66: 2024-11-23T16:15:34,193 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., storeName=0b4ea01214996120537d91f0bab5da66/A, priority=13, startTime=1732378533341; duration=0sec 2024-11-23T16:15:34,193 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T16:15:34,193 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0b4ea01214996120537d91f0bab5da66:A 2024-11-23T16:15:34,309 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:34,312 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411231b8ee75d80e04384b35b2b04c0e1ac89_0b4ea01214996120537d91f0bab5da66 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411231b8ee75d80e04384b35b2b04c0e1ac89_0b4ea01214996120537d91f0bab5da66 2024-11-23T16:15:34,313 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/.tmp/A/30e245e063404a0fb149ca57ff3364e2, store: [table=TestAcidGuarantees family=A region=0b4ea01214996120537d91f0bab5da66] 2024-11-23T16:15:34,314 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/.tmp/A/30e245e063404a0fb149ca57ff3364e2 is 175, key is test_row_0/A:col10/1732378533898/Put/seqid=0 2024-11-23T16:15:34,317 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742232_1408 (size=31255) 2024-11-23T16:15:34,717 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=357, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/.tmp/A/30e245e063404a0fb149ca57ff3364e2 2024-11-23T16:15:34,724 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/.tmp/B/56685a71a2ec4687a8d143aa703527f1 is 50, key is test_row_0/B:col10/1732378533898/Put/seqid=0 2024-11-23T16:15:34,728 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742233_1409 (size=12301) 2024-11-23T16:15:34,739 INFO [Thread-1464 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1464. 2024-11-23T16:15:34,739 DEBUG [Thread-1464 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-23T16:15:34,739 DEBUG [Thread-1464 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T16:15:34,739 DEBUG [Thread-1464 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T16:15:34,739 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-23T16:15:34,740 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-23T16:15:34,740 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 65 2024-11-23T16:15:34,740 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 68 2024-11-23T16:15:34,740 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 43 2024-11-23T16:15:34,740 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 70 2024-11-23T16:15:34,740 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 42 2024-11-23T16:15:34,740 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-23T16:15:34,740 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-23T16:15:34,740 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 956 2024-11-23T16:15:34,740 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2868 rows 2024-11-23T16:15:34,740 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 952 2024-11-23T16:15:34,740 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2856 rows 2024-11-23T16:15:34,740 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 957 2024-11-23T16:15:34,740 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2871 rows 2024-11-23T16:15:34,740 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 949 2024-11-23T16:15:34,740 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2847 rows 2024-11-23T16:15:34,740 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 955 2024-11-23T16:15:34,740 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2864 rows 2024-11-23T16:15:34,740 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-23T16:15:34,740 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobScanAtomicity(AcidGuaranteesTestBase.java:127) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-23T16:15:34,740 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T16:15:34,740 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T16:15:34,740 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-23T16:15:34,741 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-23T16:15:34,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] procedure2.ProcedureExecutor(1139): Stored pid=105, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-23T16:15:34,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-23T16:15:34,744 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732378534743"}]},"ts":"1732378534743"} 2024-11-23T16:15:34,745 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-23T16:15:34,745 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-23T16:15:34,746 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=106, ppid=105, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-23T16:15:34,747 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=107, ppid=106, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=0b4ea01214996120537d91f0bab5da66, UNASSIGN}] 2024-11-23T16:15:34,747 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=107, ppid=106, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=0b4ea01214996120537d91f0bab5da66, UNASSIGN 2024-11-23T16:15:34,748 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=107 updating hbase:meta row=0b4ea01214996120537d91f0bab5da66, regionState=CLOSING, regionLocation=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:34,749 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=107, ppid=106, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=0b4ea01214996120537d91f0bab5da66, UNASSIGN because future has completed 2024-11-23T16:15:34,749 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-23T16:15:34,749 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=108, ppid=107, state=RUNNABLE, hasLock=false; CloseRegionProcedure 0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375}] 2024-11-23T16:15:34,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-23T16:15:34,901 INFO [RS_CLOSE_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] handler.UnassignRegionHandler(122): Close 0b4ea01214996120537d91f0bab5da66 2024-11-23T16:15:34,901 DEBUG [RS_CLOSE_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-23T16:15:34,902 DEBUG [RS_CLOSE_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] regionserver.HRegion(1722): Closing 0b4ea01214996120537d91f0bab5da66, disabling compactions & flushes 2024-11-23T16:15:34,902 DEBUG [RS_CLOSE_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] regionserver.HRegion(1993): waiting for 0 compactions & cache flush to complete for region TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66. 2024-11-23T16:15:35,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-23T16:15:35,129 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=357 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/.tmp/B/56685a71a2ec4687a8d143aa703527f1 2024-11-23T16:15:35,134 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/.tmp/C/62e4d845c70e44f386e84c27e88b4bca is 50, key is test_row_0/C:col10/1732378533898/Put/seqid=0 2024-11-23T16:15:35,137 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742234_1410 (size=12301) 2024-11-23T16:15:35,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-23T16:15:35,538 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=357 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/.tmp/C/62e4d845c70e44f386e84c27e88b4bca 2024-11-23T16:15:35,542 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/.tmp/A/30e245e063404a0fb149ca57ff3364e2 as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/A/30e245e063404a0fb149ca57ff3364e2 2024-11-23T16:15:35,545 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/A/30e245e063404a0fb149ca57ff3364e2, entries=150, sequenceid=357, filesize=30.5 K 2024-11-23T16:15:35,546 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/.tmp/B/56685a71a2ec4687a8d143aa703527f1 as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/B/56685a71a2ec4687a8d143aa703527f1 2024-11-23T16:15:35,549 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/B/56685a71a2ec4687a8d143aa703527f1, entries=150, sequenceid=357, filesize=12.0 K 2024-11-23T16:15:35,549 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/.tmp/C/62e4d845c70e44f386e84c27e88b4bca as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/C/62e4d845c70e44f386e84c27e88b4bca 2024-11-23T16:15:35,552 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/C/62e4d845c70e44f386e84c27e88b4bca, entries=150, sequenceid=357, filesize=12.0 K 2024-11-23T16:15:35,553 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=13.42 KB/13740 for 0b4ea01214996120537d91f0bab5da66 in 1653ms, sequenceid=357, compaction requested=false 2024-11-23T16:15:35,553 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0b4ea01214996120537d91f0bab5da66: 2024-11-23T16:15:35,553 INFO [RS_CLOSE_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66. 2024-11-23T16:15:35,553 DEBUG [RS_CLOSE_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66. 2024-11-23T16:15:35,553 DEBUG [RS_CLOSE_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66. after waiting 0 ms 2024-11-23T16:15:35,553 DEBUG [RS_CLOSE_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66. 2024-11-23T16:15:35,553 INFO [RS_CLOSE_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] regionserver.HRegion(2902): Flushing 0b4ea01214996120537d91f0bab5da66 3/3 column families, dataSize=13.42 KB heapSize=40.59 KB 2024-11-23T16:15:35,553 DEBUG [RS_CLOSE_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0b4ea01214996120537d91f0bab5da66, store=A 2024-11-23T16:15:35,553 DEBUG [RS_CLOSE_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-23T16:15:35,553 DEBUG [RS_CLOSE_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0b4ea01214996120537d91f0bab5da66, store=B 2024-11-23T16:15:35,553 DEBUG [RS_CLOSE_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-23T16:15:35,553 DEBUG [RS_CLOSE_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0b4ea01214996120537d91f0bab5da66, store=C 2024-11-23T16:15:35,553 DEBUG [RS_CLOSE_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-23T16:15:35,557 DEBUG [RS_CLOSE_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411235af52fc00133424aab53c321f50c7807_0b4ea01214996120537d91f0bab5da66 is 50, key is test_row_0/A:col10/1732378534738/Put/seqid=0 2024-11-23T16:15:35,560 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742235_1411 (size=9914) 2024-11-23T16:15:35,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-23T16:15:35,961 DEBUG [RS_CLOSE_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:35,964 INFO [RS_CLOSE_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411235af52fc00133424aab53c321f50c7807_0b4ea01214996120537d91f0bab5da66 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411235af52fc00133424aab53c321f50c7807_0b4ea01214996120537d91f0bab5da66 2024-11-23T16:15:35,965 DEBUG [RS_CLOSE_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/.tmp/A/53c39ca0fe844aafb670b3f74419380b, store: [table=TestAcidGuarantees family=A region=0b4ea01214996120537d91f0bab5da66] 2024-11-23T16:15:35,966 DEBUG [RS_CLOSE_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/.tmp/A/53c39ca0fe844aafb670b3f74419380b is 175, key is test_row_0/A:col10/1732378534738/Put/seqid=0 2024-11-23T16:15:35,969 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742236_1412 (size=22561) 2024-11-23T16:15:36,369 INFO [RS_CLOSE_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=364, memsize=4.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/.tmp/A/53c39ca0fe844aafb670b3f74419380b 2024-11-23T16:15:36,375 DEBUG [RS_CLOSE_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/.tmp/B/d0b51b6fee30482da79d1562af71b088 is 50, key is test_row_0/B:col10/1732378534738/Put/seqid=0 2024-11-23T16:15:36,378 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742237_1413 (size=9857) 2024-11-23T16:15:36,778 INFO [RS_CLOSE_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=364 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/.tmp/B/d0b51b6fee30482da79d1562af71b088 2024-11-23T16:15:36,784 DEBUG [RS_CLOSE_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/.tmp/C/10dfb8384c084f1495212ce48d81a0af is 50, key is test_row_0/C:col10/1732378534738/Put/seqid=0 2024-11-23T16:15:36,787 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742238_1414 (size=9857) 2024-11-23T16:15:36,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-23T16:15:37,187 INFO [RS_CLOSE_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=364 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/.tmp/C/10dfb8384c084f1495212ce48d81a0af 2024-11-23T16:15:37,191 DEBUG [RS_CLOSE_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/.tmp/A/53c39ca0fe844aafb670b3f74419380b as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/A/53c39ca0fe844aafb670b3f74419380b 2024-11-23T16:15:37,194 INFO [RS_CLOSE_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/A/53c39ca0fe844aafb670b3f74419380b, entries=100, sequenceid=364, filesize=22.0 K 2024-11-23T16:15:37,195 DEBUG [RS_CLOSE_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/.tmp/B/d0b51b6fee30482da79d1562af71b088 as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/B/d0b51b6fee30482da79d1562af71b088 2024-11-23T16:15:37,198 INFO [RS_CLOSE_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/B/d0b51b6fee30482da79d1562af71b088, entries=100, sequenceid=364, filesize=9.6 K 2024-11-23T16:15:37,198 DEBUG [RS_CLOSE_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/.tmp/C/10dfb8384c084f1495212ce48d81a0af as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/C/10dfb8384c084f1495212ce48d81a0af 2024-11-23T16:15:37,201 INFO [RS_CLOSE_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/C/10dfb8384c084f1495212ce48d81a0af, entries=100, sequenceid=364, filesize=9.6 K 2024-11-23T16:15:37,202 INFO [RS_CLOSE_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] regionserver.HRegion(3140): Finished flush of dataSize ~13.42 KB/13740, heapSize ~40.55 KB/41520, currentSize=0 B/0 for 0b4ea01214996120537d91f0bab5da66 in 1648ms, sequenceid=364, compaction requested=true 2024-11-23T16:15:37,202 DEBUG [StoreCloser-TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/A/c16c26c373a14e1fb105eedc59b9a723, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/A/38d12e3abfb1412baa1c15149152447b, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/A/0c2a554442fb42cd97b193b345e0b8b4, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/A/95a56319c3994539be6585dc07758f69, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/A/5abdcf514da744d4b5e863ac23d34c04, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/A/26562298cc40485c858fbbc4359a1a8d, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/A/45d9fc022931492fb4f0e113a3584846, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/A/4b4d8e58d563499c8442ce94d2dac1c8, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/A/59beac914a7d47b7b53a7191f69b7973, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/A/5cb370ec2a0e4b88a4475c3d32e9e237, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/A/bf2c8d8c6ff2465f99ba0be092802e28, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/A/b1bb64b0378541a6983cdbccd3881f7a, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/A/0952134456d14209a6af590379db6463, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/A/8f908143c3094354995f5e2d8404bba2, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/A/3b1b572cdf7d4235a2e9170ced416c17, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/A/8fe6b959d04d44c1acdf9770855dbca4, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/A/fe634562ca1644929eaa7ba7d31d198b, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/A/0f783016cec5405aa300d7d3e870d897, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/A/5d610660216d4613800ca624da9ebefa, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/A/240a8e340d65488fb925a32c1a2dfd8d, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/A/f7561d02a2be4e0db1f4ac729f372ad3] to archive 2024-11-23T16:15:37,203 DEBUG [StoreCloser-TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-23T16:15:37,204 DEBUG [StoreCloser-TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/A/c16c26c373a14e1fb105eedc59b9a723 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/A/c16c26c373a14e1fb105eedc59b9a723 2024-11-23T16:15:37,205 DEBUG [StoreCloser-TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/A/38d12e3abfb1412baa1c15149152447b to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/A/38d12e3abfb1412baa1c15149152447b 2024-11-23T16:15:37,206 DEBUG [StoreCloser-TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/A/0c2a554442fb42cd97b193b345e0b8b4 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/A/0c2a554442fb42cd97b193b345e0b8b4 2024-11-23T16:15:37,207 DEBUG [StoreCloser-TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/A/95a56319c3994539be6585dc07758f69 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/A/95a56319c3994539be6585dc07758f69 2024-11-23T16:15:37,207 DEBUG [StoreCloser-TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/A/5abdcf514da744d4b5e863ac23d34c04 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/A/5abdcf514da744d4b5e863ac23d34c04 2024-11-23T16:15:37,208 DEBUG [StoreCloser-TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/A/26562298cc40485c858fbbc4359a1a8d to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/A/26562298cc40485c858fbbc4359a1a8d 2024-11-23T16:15:37,209 DEBUG [StoreCloser-TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/A/45d9fc022931492fb4f0e113a3584846 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/A/45d9fc022931492fb4f0e113a3584846 2024-11-23T16:15:37,210 DEBUG [StoreCloser-TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/A/4b4d8e58d563499c8442ce94d2dac1c8 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/A/4b4d8e58d563499c8442ce94d2dac1c8 2024-11-23T16:15:37,211 DEBUG [StoreCloser-TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/A/59beac914a7d47b7b53a7191f69b7973 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/A/59beac914a7d47b7b53a7191f69b7973 2024-11-23T16:15:37,211 DEBUG [StoreCloser-TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/A/5cb370ec2a0e4b88a4475c3d32e9e237 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/A/5cb370ec2a0e4b88a4475c3d32e9e237 2024-11-23T16:15:37,212 DEBUG [StoreCloser-TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/A/bf2c8d8c6ff2465f99ba0be092802e28 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/A/bf2c8d8c6ff2465f99ba0be092802e28 2024-11-23T16:15:37,213 DEBUG [StoreCloser-TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/A/b1bb64b0378541a6983cdbccd3881f7a to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/A/b1bb64b0378541a6983cdbccd3881f7a 2024-11-23T16:15:37,214 DEBUG [StoreCloser-TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/A/0952134456d14209a6af590379db6463 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/A/0952134456d14209a6af590379db6463 2024-11-23T16:15:37,215 DEBUG [StoreCloser-TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/A/8f908143c3094354995f5e2d8404bba2 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/A/8f908143c3094354995f5e2d8404bba2 2024-11-23T16:15:37,215 DEBUG [StoreCloser-TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/A/3b1b572cdf7d4235a2e9170ced416c17 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/A/3b1b572cdf7d4235a2e9170ced416c17 2024-11-23T16:15:37,216 DEBUG [StoreCloser-TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/A/8fe6b959d04d44c1acdf9770855dbca4 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/A/8fe6b959d04d44c1acdf9770855dbca4 2024-11-23T16:15:37,217 DEBUG [StoreCloser-TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/A/fe634562ca1644929eaa7ba7d31d198b to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/A/fe634562ca1644929eaa7ba7d31d198b 2024-11-23T16:15:37,218 DEBUG [StoreCloser-TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/A/0f783016cec5405aa300d7d3e870d897 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/A/0f783016cec5405aa300d7d3e870d897 2024-11-23T16:15:37,219 DEBUG [StoreCloser-TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/A/5d610660216d4613800ca624da9ebefa to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/A/5d610660216d4613800ca624da9ebefa 2024-11-23T16:15:37,220 DEBUG [StoreCloser-TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/A/240a8e340d65488fb925a32c1a2dfd8d to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/A/240a8e340d65488fb925a32c1a2dfd8d 2024-11-23T16:15:37,221 DEBUG [StoreCloser-TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/A/f7561d02a2be4e0db1f4ac729f372ad3 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/A/f7561d02a2be4e0db1f4ac729f372ad3 2024-11-23T16:15:37,222 DEBUG [StoreCloser-TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/B/8144d677a1c04286942a7fe81e4da595, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/B/2c9c68366e264b708b1a7b065ae4663e, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/B/5dbd82b35f6b4e58b22e846d1dc0a6c9, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/B/3cfdf5b8629c44b09f5bb64079e825e4, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/B/e17e820c42e3429cbd7ade2eb197b91a, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/B/0e7a82f77905450992d2e76e9b85cd30, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/B/144a47b3eac943cc96fdb9f169a47d6c, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/B/00f659f3e9d44b2cb9249f39840dc25a, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/B/5d7b6e4cfd954729845e022adb58804e, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/B/1560e1e7552a449dae570a2934b12e5b, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/B/e0ce66cdc1084169ad0e623cdb557b70, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/B/c56aaef115a94f7aa1a8d975fc676cc9, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/B/f92848caab9a4196b331fa99e97d2dab, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/B/b0391390cad647918a967ef86fb57475, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/B/4ff6b8de730b416cbde479cdfa30b570, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/B/9404e71d2dad4911bb3a0a1e22f54cbd, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/B/610eb10921f54452b7dcc45c2fad22e1, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/B/25608670fa694a57997894b6648a844e, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/B/cd2729ddd6c846e8aeb90dd926d2790e, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/B/dbb79f5eb7c14870bef5b1ab5d57ce3d, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/B/7a03aac3067e4f678940ab196181f5e3] to archive 2024-11-23T16:15:37,223 DEBUG [StoreCloser-TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-23T16:15:37,224 DEBUG [StoreCloser-TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/B/8144d677a1c04286942a7fe81e4da595 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/B/8144d677a1c04286942a7fe81e4da595 2024-11-23T16:15:37,225 DEBUG [StoreCloser-TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/B/2c9c68366e264b708b1a7b065ae4663e to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/B/2c9c68366e264b708b1a7b065ae4663e 2024-11-23T16:15:37,226 DEBUG [StoreCloser-TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/B/5dbd82b35f6b4e58b22e846d1dc0a6c9 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/B/5dbd82b35f6b4e58b22e846d1dc0a6c9 2024-11-23T16:15:37,226 DEBUG [StoreCloser-TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/B/3cfdf5b8629c44b09f5bb64079e825e4 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/B/3cfdf5b8629c44b09f5bb64079e825e4 2024-11-23T16:15:37,227 DEBUG [StoreCloser-TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/B/e17e820c42e3429cbd7ade2eb197b91a to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/B/e17e820c42e3429cbd7ade2eb197b91a 2024-11-23T16:15:37,228 DEBUG [StoreCloser-TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/B/0e7a82f77905450992d2e76e9b85cd30 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/B/0e7a82f77905450992d2e76e9b85cd30 2024-11-23T16:15:37,228 DEBUG [StoreCloser-TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/B/144a47b3eac943cc96fdb9f169a47d6c to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/B/144a47b3eac943cc96fdb9f169a47d6c 2024-11-23T16:15:37,229 DEBUG [StoreCloser-TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/B/00f659f3e9d44b2cb9249f39840dc25a to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/B/00f659f3e9d44b2cb9249f39840dc25a 2024-11-23T16:15:37,230 DEBUG [StoreCloser-TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/B/5d7b6e4cfd954729845e022adb58804e to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/B/5d7b6e4cfd954729845e022adb58804e 2024-11-23T16:15:37,230 DEBUG [StoreCloser-TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/B/1560e1e7552a449dae570a2934b12e5b to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/B/1560e1e7552a449dae570a2934b12e5b 2024-11-23T16:15:37,231 DEBUG [StoreCloser-TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/B/e0ce66cdc1084169ad0e623cdb557b70 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/B/e0ce66cdc1084169ad0e623cdb557b70 2024-11-23T16:15:37,232 DEBUG [StoreCloser-TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/B/c56aaef115a94f7aa1a8d975fc676cc9 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/B/c56aaef115a94f7aa1a8d975fc676cc9 2024-11-23T16:15:37,232 DEBUG [StoreCloser-TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/B/f92848caab9a4196b331fa99e97d2dab to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/B/f92848caab9a4196b331fa99e97d2dab 2024-11-23T16:15:37,233 DEBUG [StoreCloser-TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/B/b0391390cad647918a967ef86fb57475 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/B/b0391390cad647918a967ef86fb57475 2024-11-23T16:15:37,234 DEBUG [StoreCloser-TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/B/4ff6b8de730b416cbde479cdfa30b570 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/B/4ff6b8de730b416cbde479cdfa30b570 2024-11-23T16:15:37,234 DEBUG [StoreCloser-TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/B/9404e71d2dad4911bb3a0a1e22f54cbd to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/B/9404e71d2dad4911bb3a0a1e22f54cbd 2024-11-23T16:15:37,235 DEBUG [StoreCloser-TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/B/610eb10921f54452b7dcc45c2fad22e1 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/B/610eb10921f54452b7dcc45c2fad22e1 2024-11-23T16:15:37,236 DEBUG [StoreCloser-TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/B/25608670fa694a57997894b6648a844e to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/B/25608670fa694a57997894b6648a844e 2024-11-23T16:15:37,237 DEBUG [StoreCloser-TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/B/cd2729ddd6c846e8aeb90dd926d2790e to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/B/cd2729ddd6c846e8aeb90dd926d2790e 2024-11-23T16:15:37,237 DEBUG [StoreCloser-TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/B/dbb79f5eb7c14870bef5b1ab5d57ce3d to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/B/dbb79f5eb7c14870bef5b1ab5d57ce3d 2024-11-23T16:15:37,238 DEBUG [StoreCloser-TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/B/7a03aac3067e4f678940ab196181f5e3 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/B/7a03aac3067e4f678940ab196181f5e3 2024-11-23T16:15:37,239 DEBUG [StoreCloser-TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/C/7ad28d3a58e74aee953d1f7711776cb8, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/C/ef5cf31c695d4057af66caabf14a5b5f, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/C/2d663f8abde74ba9b95108466a94a376, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/C/1a071b2a0b464c38a2d197aba97025ed, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/C/3665238e9bb74ed0b90676dfdcb39625, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/C/0aa96d0e877843d6a90e2557993e71b5, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/C/24d7117c8c274bbc90bfdd548d9714bb, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/C/d9293febc9d449adb396df71274727ee, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/C/5af48329ede3475d86cf43ecc94f84ed, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/C/e9558cf58a984213b55b0088f5e7b2d9, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/C/5debbf60369542a6bee33065de8007f9, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/C/331031eb35fc44d9a8ebeaf7c8de922c, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/C/1baf74d4d9ee4fd983b07941f20e80d8, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/C/1094bffb05654d9487b9c5d5168fcdfc, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/C/cf8bbe7d16204bdaa6a41a337821c588, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/C/8d6b85562863486e9b61551a77d38369, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/C/85b4a4b5f67f4bceb5f9297497435fba, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/C/0d79d4e151d541dd9cd4c4d81844bf99, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/C/257345c6b0e04480850d4a9c3bd0b6e2, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/C/9ab3f9205d1245489a754d369f2a3700, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/C/1dd00b256cf443d2978113ab2031fff7] to archive 2024-11-23T16:15:37,240 DEBUG [StoreCloser-TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-23T16:15:37,241 DEBUG [StoreCloser-TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/C/7ad28d3a58e74aee953d1f7711776cb8 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/C/7ad28d3a58e74aee953d1f7711776cb8 2024-11-23T16:15:37,242 DEBUG [StoreCloser-TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/C/ef5cf31c695d4057af66caabf14a5b5f to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/C/ef5cf31c695d4057af66caabf14a5b5f 2024-11-23T16:15:37,242 DEBUG [StoreCloser-TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/C/2d663f8abde74ba9b95108466a94a376 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/C/2d663f8abde74ba9b95108466a94a376 2024-11-23T16:15:37,243 DEBUG [StoreCloser-TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/C/1a071b2a0b464c38a2d197aba97025ed to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/C/1a071b2a0b464c38a2d197aba97025ed 2024-11-23T16:15:37,244 DEBUG [StoreCloser-TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/C/3665238e9bb74ed0b90676dfdcb39625 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/C/3665238e9bb74ed0b90676dfdcb39625 2024-11-23T16:15:37,245 DEBUG [StoreCloser-TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/C/0aa96d0e877843d6a90e2557993e71b5 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/C/0aa96d0e877843d6a90e2557993e71b5 2024-11-23T16:15:37,245 DEBUG [StoreCloser-TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/C/24d7117c8c274bbc90bfdd548d9714bb to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/C/24d7117c8c274bbc90bfdd548d9714bb 2024-11-23T16:15:37,246 DEBUG [StoreCloser-TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/C/d9293febc9d449adb396df71274727ee to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/C/d9293febc9d449adb396df71274727ee 2024-11-23T16:15:37,247 DEBUG [StoreCloser-TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/C/5af48329ede3475d86cf43ecc94f84ed to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/C/5af48329ede3475d86cf43ecc94f84ed 2024-11-23T16:15:37,248 DEBUG [StoreCloser-TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/C/e9558cf58a984213b55b0088f5e7b2d9 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/C/e9558cf58a984213b55b0088f5e7b2d9 2024-11-23T16:15:37,249 DEBUG [StoreCloser-TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/C/5debbf60369542a6bee33065de8007f9 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/C/5debbf60369542a6bee33065de8007f9 2024-11-23T16:15:37,249 DEBUG [StoreCloser-TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/C/331031eb35fc44d9a8ebeaf7c8de922c to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/C/331031eb35fc44d9a8ebeaf7c8de922c 2024-11-23T16:15:37,250 DEBUG [StoreCloser-TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/C/1baf74d4d9ee4fd983b07941f20e80d8 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/C/1baf74d4d9ee4fd983b07941f20e80d8 2024-11-23T16:15:37,251 DEBUG [StoreCloser-TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/C/1094bffb05654d9487b9c5d5168fcdfc to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/C/1094bffb05654d9487b9c5d5168fcdfc 2024-11-23T16:15:37,252 DEBUG [StoreCloser-TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/C/cf8bbe7d16204bdaa6a41a337821c588 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/C/cf8bbe7d16204bdaa6a41a337821c588 2024-11-23T16:15:37,252 DEBUG [StoreCloser-TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/C/8d6b85562863486e9b61551a77d38369 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/C/8d6b85562863486e9b61551a77d38369 2024-11-23T16:15:37,253 DEBUG [StoreCloser-TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/C/85b4a4b5f67f4bceb5f9297497435fba to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/C/85b4a4b5f67f4bceb5f9297497435fba 2024-11-23T16:15:37,254 DEBUG [StoreCloser-TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/C/0d79d4e151d541dd9cd4c4d81844bf99 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/C/0d79d4e151d541dd9cd4c4d81844bf99 2024-11-23T16:15:37,255 DEBUG [StoreCloser-TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/C/257345c6b0e04480850d4a9c3bd0b6e2 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/C/257345c6b0e04480850d4a9c3bd0b6e2 2024-11-23T16:15:37,255 DEBUG [StoreCloser-TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/C/9ab3f9205d1245489a754d369f2a3700 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/C/9ab3f9205d1245489a754d369f2a3700 2024-11-23T16:15:37,256 DEBUG [StoreCloser-TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/C/1dd00b256cf443d2978113ab2031fff7 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/C/1dd00b256cf443d2978113ab2031fff7 2024-11-23T16:15:37,260 DEBUG [RS_CLOSE_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/recovered.edits/367.seqid, newMaxSeqId=367, maxSeqId=4 2024-11-23T16:15:37,260 INFO [RS_CLOSE_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66. 2024-11-23T16:15:37,260 DEBUG [RS_CLOSE_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] regionserver.HRegion(1676): Region close journal for 0b4ea01214996120537d91f0bab5da66: Waiting for close lock at 1732378534901Running coprocessor pre-close hooks at 1732378534901Disabling compacts and flushes for region at 1732378534901Disabling writes for close at 1732378535553 (+652 ms)Obtaining lock to block concurrent updates at 1732378535553Preparing flush snapshotting stores in 0b4ea01214996120537d91f0bab5da66 at 1732378535553Finished memstore snapshotting TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66., syncing WAL and waiting on mvcc, flushsize=dataSize=13740, getHeapSize=41520, getOffHeapSize=0, getCellsCount=300 at 1732378535553Flushing stores of TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66. at 1732378535553Flushing 0b4ea01214996120537d91f0bab5da66/A: creating writer at 1732378535553Flushing mob file 0b4ea01214996120537d91f0bab5da66/A: appending metadata at 1732378535557 (+4 ms)Flushing mob file 0b4ea01214996120537d91f0bab5da66/A: closing flushed file at 1732378535557Flushing 0b4ea01214996120537d91f0bab5da66/A: appending metadata at 1732378535965 (+408 ms)Flushing 0b4ea01214996120537d91f0bab5da66/A: closing flushed file at 1732378535965Flushing 0b4ea01214996120537d91f0bab5da66/B: creating writer at 1732378536373 (+408 ms)Flushing 0b4ea01214996120537d91f0bab5da66/B: appending metadata at 1732378536374 (+1 ms)Flushing 0b4ea01214996120537d91f0bab5da66/B: closing flushed file at 1732378536375 (+1 ms)Flushing 0b4ea01214996120537d91f0bab5da66/C: creating writer at 1732378536781 (+406 ms)Flushing 0b4ea01214996120537d91f0bab5da66/C: appending metadata at 1732378536783 (+2 ms)Flushing 0b4ea01214996120537d91f0bab5da66/C: closing flushed file at 1732378536783Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@38ec663f: reopening flushed file at 1732378537191 (+408 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@150ab981: reopening flushed file at 1732378537194 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@70ecc5f8: reopening flushed file at 1732378537198 (+4 ms)Finished flush of dataSize ~13.42 KB/13740, heapSize ~40.55 KB/41520, currentSize=0 B/0 for 0b4ea01214996120537d91f0bab5da66 in 1648ms, sequenceid=364, compaction requested=true at 1732378537202 (+4 ms)Writing region close event to WAL at 1732378537257 (+55 ms)Running coprocessor post-close hooks at 1732378537260 (+3 ms)Closed at 1732378537260 2024-11-23T16:15:37,262 INFO [RS_CLOSE_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] handler.UnassignRegionHandler(157): Closed 0b4ea01214996120537d91f0bab5da66 2024-11-23T16:15:37,262 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=107 updating hbase:meta row=0b4ea01214996120537d91f0bab5da66, regionState=CLOSED 2024-11-23T16:15:37,263 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=108, ppid=107, state=RUNNABLE, hasLock=false; CloseRegionProcedure 0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 because future has completed 2024-11-23T16:15:37,265 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=108, resume processing ppid=107 2024-11-23T16:15:37,265 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=108, ppid=107, state=SUCCESS, hasLock=false; CloseRegionProcedure 0b4ea01214996120537d91f0bab5da66, server=1a7c5470b472,44715,1732378420375 in 2.5150 sec 2024-11-23T16:15:37,266 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=107, resume processing ppid=106 2024-11-23T16:15:37,266 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=107, ppid=106, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=0b4ea01214996120537d91f0bab5da66, UNASSIGN in 2.5180 sec 2024-11-23T16:15:37,268 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=106, resume processing ppid=105 2024-11-23T16:15:37,268 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=106, ppid=105, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 2.5210 sec 2024-11-23T16:15:37,269 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732378537269"}]},"ts":"1732378537269"} 2024-11-23T16:15:37,270 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-23T16:15:37,270 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-23T16:15:37,271 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=105, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 2.5300 sec 2024-11-23T16:15:38,631 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-23T16:15:38,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-23T16:15:38,898 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-23T16:15:38,898 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-23T16:15:38,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] procedure2.ProcedureExecutor(1139): Stored pid=109, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-23T16:15:38,900 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=109, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-23T16:15:38,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-23T16:15:38,900 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=109, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-23T16:15:38,902 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66 2024-11-23T16:15:38,903 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/A, FileablePath, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/B, FileablePath, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/C, FileablePath, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/recovered.edits] 2024-11-23T16:15:38,905 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/A/30e245e063404a0fb149ca57ff3364e2 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/A/30e245e063404a0fb149ca57ff3364e2 2024-11-23T16:15:38,906 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/A/53c39ca0fe844aafb670b3f74419380b to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/A/53c39ca0fe844aafb670b3f74419380b 2024-11-23T16:15:38,907 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/A/f0f556b04fb841acaec352677c1e4c24 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/A/f0f556b04fb841acaec352677c1e4c24 2024-11-23T16:15:38,909 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/B/56685a71a2ec4687a8d143aa703527f1 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/B/56685a71a2ec4687a8d143aa703527f1 2024-11-23T16:15:38,910 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/B/7f84e325eaa44fc2a37509f40d0c9cf9 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/B/7f84e325eaa44fc2a37509f40d0c9cf9 2024-11-23T16:15:38,910 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/B/d0b51b6fee30482da79d1562af71b088 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/B/d0b51b6fee30482da79d1562af71b088 2024-11-23T16:15:38,912 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/C/10dfb8384c084f1495212ce48d81a0af to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/C/10dfb8384c084f1495212ce48d81a0af 2024-11-23T16:15:38,913 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/C/62e4d845c70e44f386e84c27e88b4bca to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/C/62e4d845c70e44f386e84c27e88b4bca 2024-11-23T16:15:38,914 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/C/8469622c02294afe8d2b7b1191b84c21 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/C/8469622c02294afe8d2b7b1191b84c21 2024-11-23T16:15:38,915 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/recovered.edits/367.seqid to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66/recovered.edits/367.seqid 2024-11-23T16:15:38,916 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/0b4ea01214996120537d91f0bab5da66 2024-11-23T16:15:38,916 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-23T16:15:38,916 DEBUG [PEWorker-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-23T16:15:38,917 DEBUG [PEWorker-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-23T16:15:38,919 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112307454ec926ca4cbd95680fb2d146d0ca_0b4ea01214996120537d91f0bab5da66 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112307454ec926ca4cbd95680fb2d146d0ca_0b4ea01214996120537d91f0bab5da66 2024-11-23T16:15:38,920 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411230a5a65e2e1e741f4b85db6f997ef688e_0b4ea01214996120537d91f0bab5da66 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411230a5a65e2e1e741f4b85db6f997ef688e_0b4ea01214996120537d91f0bab5da66 2024-11-23T16:15:38,921 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112317bbd387ba2749e9bf4c050a05c44f4c_0b4ea01214996120537d91f0bab5da66 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112317bbd387ba2749e9bf4c050a05c44f4c_0b4ea01214996120537d91f0bab5da66 2024-11-23T16:15:38,922 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411231b8ee75d80e04384b35b2b04c0e1ac89_0b4ea01214996120537d91f0bab5da66 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411231b8ee75d80e04384b35b2b04c0e1ac89_0b4ea01214996120537d91f0bab5da66 2024-11-23T16:15:38,923 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411232f772b236d29426fa9f4e3049561c9fd_0b4ea01214996120537d91f0bab5da66 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411232f772b236d29426fa9f4e3049561c9fd_0b4ea01214996120537d91f0bab5da66 2024-11-23T16:15:38,923 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112351c0a7908f6240e69b1bf96880325bd1_0b4ea01214996120537d91f0bab5da66 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112351c0a7908f6240e69b1bf96880325bd1_0b4ea01214996120537d91f0bab5da66 2024-11-23T16:15:38,924 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411235805391473964cbcaade71fd7c21caf1_0b4ea01214996120537d91f0bab5da66 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411235805391473964cbcaade71fd7c21caf1_0b4ea01214996120537d91f0bab5da66 2024-11-23T16:15:38,925 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411235af52fc00133424aab53c321f50c7807_0b4ea01214996120537d91f0bab5da66 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411235af52fc00133424aab53c321f50c7807_0b4ea01214996120537d91f0bab5da66 2024-11-23T16:15:38,926 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123659e4cf4a9934db9b2d5ecea34c83d45_0b4ea01214996120537d91f0bab5da66 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123659e4cf4a9934db9b2d5ecea34c83d45_0b4ea01214996120537d91f0bab5da66 2024-11-23T16:15:38,927 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123666d3b50debc4fa09e897cab2daa2188_0b4ea01214996120537d91f0bab5da66 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123666d3b50debc4fa09e897cab2daa2188_0b4ea01214996120537d91f0bab5da66 2024-11-23T16:15:38,928 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112367c4c16b30524e10903209650d1c819b_0b4ea01214996120537d91f0bab5da66 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112367c4c16b30524e10903209650d1c819b_0b4ea01214996120537d91f0bab5da66 2024-11-23T16:15:38,928 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411236e7c8e9ccc044179b96b3a549ff9aec5_0b4ea01214996120537d91f0bab5da66 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411236e7c8e9ccc044179b96b3a549ff9aec5_0b4ea01214996120537d91f0bab5da66 2024-11-23T16:15:38,929 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112375a02a6cc2764cb5a1c9225b7a9767ed_0b4ea01214996120537d91f0bab5da66 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112375a02a6cc2764cb5a1c9225b7a9767ed_0b4ea01214996120537d91f0bab5da66 2024-11-23T16:15:38,930 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112376fb74abada1481eaac8f227110edb8e_0b4ea01214996120537d91f0bab5da66 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112376fb74abada1481eaac8f227110edb8e_0b4ea01214996120537d91f0bab5da66 2024-11-23T16:15:38,931 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123ad776715dfe64847ac99d06475270914_0b4ea01214996120537d91f0bab5da66 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123ad776715dfe64847ac99d06475270914_0b4ea01214996120537d91f0bab5da66 2024-11-23T16:15:38,932 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123ce3ce2ed690244d181a2bd65f9ba5458_0b4ea01214996120537d91f0bab5da66 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123ce3ce2ed690244d181a2bd65f9ba5458_0b4ea01214996120537d91f0bab5da66 2024-11-23T16:15:38,933 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123dbb3b61e507c4c4abe3dcdcc6667191e_0b4ea01214996120537d91f0bab5da66 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123dbb3b61e507c4c4abe3dcdcc6667191e_0b4ea01214996120537d91f0bab5da66 2024-11-23T16:15:38,934 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123fd2eb836b9bb478aa74acb95866049c3_0b4ea01214996120537d91f0bab5da66 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123fd2eb836b9bb478aa74acb95866049c3_0b4ea01214996120537d91f0bab5da66 2024-11-23T16:15:38,934 DEBUG [PEWorker-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-23T16:15:38,936 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=109, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-23T16:15:38,938 WARN [PEWorker-2 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-23T16:15:38,940 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-23T16:15:38,941 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=109, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-23T16:15:38,941 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-23T16:15:38,941 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732378538941"}]},"ts":"9223372036854775807"} 2024-11-23T16:15:38,943 INFO [PEWorker-2 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-23T16:15:38,943 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 0b4ea01214996120537d91f0bab5da66, NAME => 'TestAcidGuarantees,,1732378509490.0b4ea01214996120537d91f0bab5da66.', STARTKEY => '', ENDKEY => ''}] 2024-11-23T16:15:38,943 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-23T16:15:38,943 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732378538943"}]},"ts":"9223372036854775807"} 2024-11-23T16:15:38,944 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-23T16:15:38,945 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(133): Finished pid=109, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-23T16:15:38,946 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=109, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 47 msec 2024-11-23T16:15:39,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-23T16:15:39,008 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-23T16:15:39,008 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-23T16:15:39,017 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobScanAtomicity Thread=251 (was 248) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-2057066486_22 at /127.0.0.1:60896 [Waiting for operation #311] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1268241868_22 at /127.0.0.1:44938 [Waiting for operation #267] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1268241868_22 at /127.0.0.1:44962 [Waiting for operation #256] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-2057066486_22 at /127.0.0.1:44954 [Waiting for operation #266] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=459 (was 451) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=386 (was 404), ProcessCount=11 (was 11), AvailableMemoryMB=6810 (was 7305) 2024-11-23T16:15:39,026 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testScanAtomicity Thread=251, OpenFileDescriptor=459, MaxFileDescriptor=1048576, SystemLoadAverage=386, ProcessCount=11, AvailableMemoryMB=6810 2024-11-23T16:15:39,027 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-23T16:15:39,028 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-23T16:15:39,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] procedure2.ProcedureExecutor(1139): Stored pid=110, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-23T16:15:39,030 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=110, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-23T16:15:39,030 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:39,030 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 110 2024-11-23T16:15:39,031 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=110, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-23T16:15:39,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=110 2024-11-23T16:15:39,035 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742239_1415 (size=1061) 2024-11-23T16:15:39,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=110 2024-11-23T16:15:39,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=110 2024-11-23T16:15:39,437 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 63f43c63344ece03d182c42497a352e4, NAME => 'TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca 2024-11-23T16:15:39,441 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742240_1416 (size=53) 2024-11-23T16:15:39,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=110 2024-11-23T16:15:39,842 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T16:15:39,842 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 63f43c63344ece03d182c42497a352e4, disabling compactions & flushes 2024-11-23T16:15:39,842 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4. 2024-11-23T16:15:39,842 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4. 2024-11-23T16:15:39,842 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4. after waiting 0 ms 2024-11-23T16:15:39,842 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4. 2024-11-23T16:15:39,842 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4. 2024-11-23T16:15:39,842 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 63f43c63344ece03d182c42497a352e4: Waiting for close lock at 1732378539842Disabling compacts and flushes for region at 1732378539842Disabling writes for close at 1732378539842Writing region close event to WAL at 1732378539842Closed at 1732378539842 2024-11-23T16:15:39,843 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=110, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-23T16:15:39,843 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732378539843"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732378539843"}]},"ts":"1732378539843"} 2024-11-23T16:15:39,845 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-23T16:15:39,845 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=110, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-23T16:15:39,845 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732378539845"}]},"ts":"1732378539845"} 2024-11-23T16:15:39,846 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-23T16:15:39,847 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=111, ppid=110, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=63f43c63344ece03d182c42497a352e4, ASSIGN}] 2024-11-23T16:15:39,847 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=111, ppid=110, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=63f43c63344ece03d182c42497a352e4, ASSIGN 2024-11-23T16:15:39,848 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=111, ppid=110, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=63f43c63344ece03d182c42497a352e4, ASSIGN; state=OFFLINE, location=1a7c5470b472,44715,1732378420375; forceNewPlan=false, retain=false 2024-11-23T16:15:39,999 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=111 updating hbase:meta row=63f43c63344ece03d182c42497a352e4, regionState=OPENING, regionLocation=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:40,000 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=111, ppid=110, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=63f43c63344ece03d182c42497a352e4, ASSIGN because future has completed 2024-11-23T16:15:40,000 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=112, ppid=111, state=RUNNABLE, hasLock=false; OpenRegionProcedure 63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375}] 2024-11-23T16:15:40,108 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-23T16:15:40,155 INFO [RS_OPEN_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_OPEN_REGION, pid=112}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4. 2024-11-23T16:15:40,155 DEBUG [RS_OPEN_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_OPEN_REGION, pid=112}] regionserver.HRegion(7752): Opening region: {ENCODED => 63f43c63344ece03d182c42497a352e4, NAME => 'TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4.', STARTKEY => '', ENDKEY => ''} 2024-11-23T16:15:40,156 DEBUG [RS_OPEN_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_OPEN_REGION, pid=112}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 63f43c63344ece03d182c42497a352e4 2024-11-23T16:15:40,156 DEBUG [RS_OPEN_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_OPEN_REGION, pid=112}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T16:15:40,156 DEBUG [RS_OPEN_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_OPEN_REGION, pid=112}] regionserver.HRegion(7794): checking encryption for 63f43c63344ece03d182c42497a352e4 2024-11-23T16:15:40,156 DEBUG [RS_OPEN_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_OPEN_REGION, pid=112}] regionserver.HRegion(7797): checking classloading for 63f43c63344ece03d182c42497a352e4 2024-11-23T16:15:40,157 INFO [StoreOpener-63f43c63344ece03d182c42497a352e4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 63f43c63344ece03d182c42497a352e4 2024-11-23T16:15:40,158 INFO [StoreOpener-63f43c63344ece03d182c42497a352e4-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-23T16:15:40,158 INFO [StoreOpener-63f43c63344ece03d182c42497a352e4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 63f43c63344ece03d182c42497a352e4 columnFamilyName A 2024-11-23T16:15:40,158 DEBUG [StoreOpener-63f43c63344ece03d182c42497a352e4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:40,159 INFO [StoreOpener-63f43c63344ece03d182c42497a352e4-1 {}] regionserver.HStore(327): Store=63f43c63344ece03d182c42497a352e4/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T16:15:40,159 INFO [StoreOpener-63f43c63344ece03d182c42497a352e4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 63f43c63344ece03d182c42497a352e4 2024-11-23T16:15:40,160 INFO [StoreOpener-63f43c63344ece03d182c42497a352e4-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-23T16:15:40,160 INFO [StoreOpener-63f43c63344ece03d182c42497a352e4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 63f43c63344ece03d182c42497a352e4 columnFamilyName B 2024-11-23T16:15:40,160 DEBUG [StoreOpener-63f43c63344ece03d182c42497a352e4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:40,160 INFO [StoreOpener-63f43c63344ece03d182c42497a352e4-1 {}] regionserver.HStore(327): Store=63f43c63344ece03d182c42497a352e4/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T16:15:40,160 INFO [StoreOpener-63f43c63344ece03d182c42497a352e4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 63f43c63344ece03d182c42497a352e4 2024-11-23T16:15:40,161 INFO [StoreOpener-63f43c63344ece03d182c42497a352e4-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-23T16:15:40,161 INFO [StoreOpener-63f43c63344ece03d182c42497a352e4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 63f43c63344ece03d182c42497a352e4 columnFamilyName C 2024-11-23T16:15:40,161 DEBUG [StoreOpener-63f43c63344ece03d182c42497a352e4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:15:40,162 INFO [StoreOpener-63f43c63344ece03d182c42497a352e4-1 {}] regionserver.HStore(327): Store=63f43c63344ece03d182c42497a352e4/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T16:15:40,162 INFO [RS_OPEN_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_OPEN_REGION, pid=112}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4. 2024-11-23T16:15:40,162 DEBUG [RS_OPEN_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_OPEN_REGION, pid=112}] regionserver.HRegion(1038): replaying wal for 63f43c63344ece03d182c42497a352e4 2024-11-23T16:15:40,162 DEBUG [RS_OPEN_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_OPEN_REGION, pid=112}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4 2024-11-23T16:15:40,163 DEBUG [RS_OPEN_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_OPEN_REGION, pid=112}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4 2024-11-23T16:15:40,163 DEBUG [RS_OPEN_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_OPEN_REGION, pid=112}] regionserver.HRegion(1048): stopping wal replay for 63f43c63344ece03d182c42497a352e4 2024-11-23T16:15:40,163 DEBUG [RS_OPEN_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_OPEN_REGION, pid=112}] regionserver.HRegion(1060): Cleaning up temporary data for 63f43c63344ece03d182c42497a352e4 2024-11-23T16:15:40,164 DEBUG [RS_OPEN_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_OPEN_REGION, pid=112}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-23T16:15:40,164 DEBUG [RS_OPEN_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_OPEN_REGION, pid=112}] regionserver.HRegion(1093): writing seq id for 63f43c63344ece03d182c42497a352e4 2024-11-23T16:15:40,166 DEBUG [RS_OPEN_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_OPEN_REGION, pid=112}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-23T16:15:40,166 INFO [RS_OPEN_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_OPEN_REGION, pid=112}] regionserver.HRegion(1114): Opened 63f43c63344ece03d182c42497a352e4; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68696293, jitterRate=0.02365453541278839}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-23T16:15:40,166 DEBUG [RS_OPEN_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_OPEN_REGION, pid=112}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 63f43c63344ece03d182c42497a352e4 2024-11-23T16:15:40,167 DEBUG [RS_OPEN_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_OPEN_REGION, pid=112}] regionserver.HRegion(1006): Region open journal for 63f43c63344ece03d182c42497a352e4: Running coprocessor pre-open hook at 1732378540156Writing region info on filesystem at 1732378540156Initializing all the Stores at 1732378540157 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732378540157Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732378540157Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732378540157Cleaning up temporary data from old regions at 1732378540163 (+6 ms)Running coprocessor post-open hooks at 1732378540166 (+3 ms)Region opened successfully at 1732378540167 (+1 ms) 2024-11-23T16:15:40,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=110 2024-11-23T16:15:40,167 INFO [RS_OPEN_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_OPEN_REGION, pid=112}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., pid=112, masterSystemTime=1732378540152 2024-11-23T16:15:40,169 DEBUG [RS_OPEN_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_OPEN_REGION, pid=112}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4. 2024-11-23T16:15:40,169 INFO [RS_OPEN_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_OPEN_REGION, pid=112}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4. 2024-11-23T16:15:40,169 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=111 updating hbase:meta row=63f43c63344ece03d182c42497a352e4, regionState=OPEN, openSeqNum=2, regionLocation=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:40,171 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=112, ppid=111, state=RUNNABLE, hasLock=false; OpenRegionProcedure 63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 because future has completed 2024-11-23T16:15:40,172 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=112, resume processing ppid=111 2024-11-23T16:15:40,172 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=112, ppid=111, state=SUCCESS, hasLock=false; OpenRegionProcedure 63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 in 171 msec 2024-11-23T16:15:40,174 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=111, resume processing ppid=110 2024-11-23T16:15:40,174 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=111, ppid=110, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=63f43c63344ece03d182c42497a352e4, ASSIGN in 325 msec 2024-11-23T16:15:40,174 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=110, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-23T16:15:40,175 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732378540174"}]},"ts":"1732378540174"} 2024-11-23T16:15:40,175 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-23T16:15:40,176 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=110, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-23T16:15:40,177 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=110, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1480 sec 2024-11-23T16:15:41,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=110 2024-11-23T16:15:41,188 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-23T16:15:41,189 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@55d1f058, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T16:15:41,189 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1a7c5470b472,45799,-1 for getting cluster id 2024-11-23T16:15:41,189 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-23T16:15:41,190 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '34e0592b-548a-4d73-be6a-df1bafadfd92' 2024-11-23T16:15:41,190 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-23T16:15:41,190 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "34e0592b-548a-4d73-be6a-df1bafadfd92" 2024-11-23T16:15:41,191 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1bdb284, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T16:15:41,191 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1a7c5470b472,45799,-1] 2024-11-23T16:15:41,191 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-23T16:15:41,191 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T16:15:41,192 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58502, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-23T16:15:41,192 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@17e7d31d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T16:15:41,192 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T16:15:41,193 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1a7c5470b472,44715,1732378420375, seqNum=-1] 2024-11-23T16:15:41,193 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T16:15:41,194 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39710, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T16:15:41,195 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-23T16:15:41,195 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncConnectionImpl(321): The fetched master address is 1a7c5470b472,45799,1732378419617 2024-11-23T16:15:41,195 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@5d6253d2 2024-11-23T16:15:41,195 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-23T16:15:41,196 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58506, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-23T16:15:41,197 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@581c4bd0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T16:15:41,197 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1a7c5470b472,45799,-1 for getting cluster id 2024-11-23T16:15:41,197 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-23T16:15:41,198 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '34e0592b-548a-4d73-be6a-df1bafadfd92' 2024-11-23T16:15:41,198 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-23T16:15:41,198 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "34e0592b-548a-4d73-be6a-df1bafadfd92" 2024-11-23T16:15:41,198 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@603dc8b3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T16:15:41,198 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1a7c5470b472,45799,-1] 2024-11-23T16:15:41,198 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-23T16:15:41,198 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T16:15:41,199 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58530, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-23T16:15:41,199 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@76903a7b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T16:15:41,200 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3692791b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T16:15:41,200 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1a7c5470b472,45799,-1 for getting cluster id 2024-11-23T16:15:41,200 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-23T16:15:41,200 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '34e0592b-548a-4d73-be6a-df1bafadfd92' 2024-11-23T16:15:41,200 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-23T16:15:41,200 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "34e0592b-548a-4d73-be6a-df1bafadfd92" 2024-11-23T16:15:41,200 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1a70c81f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T16:15:41,200 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1a7c5470b472,45799,-1] 2024-11-23T16:15:41,201 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-23T16:15:41,201 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T16:15:41,201 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58544, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-23T16:15:41,202 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@65f9bbdd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T16:15:41,202 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2b791feb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T16:15:41,202 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1a7c5470b472,45799,-1 for getting cluster id 2024-11-23T16:15:41,202 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-23T16:15:41,203 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '34e0592b-548a-4d73-be6a-df1bafadfd92' 2024-11-23T16:15:41,203 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-23T16:15:41,203 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "34e0592b-548a-4d73-be6a-df1bafadfd92" 2024-11-23T16:15:41,203 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3c2cc5d1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T16:15:41,203 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1a7c5470b472,45799,-1] 2024-11-23T16:15:41,203 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-23T16:15:41,203 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T16:15:41,204 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58570, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-23T16:15:41,204 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@37a563bc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T16:15:41,204 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6da2e0be, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T16:15:41,204 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1a7c5470b472,45799,-1 for getting cluster id 2024-11-23T16:15:41,204 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-23T16:15:41,205 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '34e0592b-548a-4d73-be6a-df1bafadfd92' 2024-11-23T16:15:41,205 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-23T16:15:41,205 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "34e0592b-548a-4d73-be6a-df1bafadfd92" 2024-11-23T16:15:41,205 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@25638c95, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T16:15:41,205 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1a7c5470b472,45799,-1] 2024-11-23T16:15:41,205 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-23T16:15:41,205 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T16:15:41,206 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58580, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-23T16:15:41,206 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@33b17fcd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T16:15:41,207 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3fc361f0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T16:15:41,207 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1a7c5470b472,45799,-1 for getting cluster id 2024-11-23T16:15:41,207 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-23T16:15:41,207 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '34e0592b-548a-4d73-be6a-df1bafadfd92' 2024-11-23T16:15:41,207 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-23T16:15:41,207 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "34e0592b-548a-4d73-be6a-df1bafadfd92" 2024-11-23T16:15:41,207 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@8d2517f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T16:15:41,208 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1a7c5470b472,45799,-1] 2024-11-23T16:15:41,208 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-23T16:15:41,208 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T16:15:41,208 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58590, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-23T16:15:41,209 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2260a5b4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T16:15:41,209 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f59c6a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T16:15:41,209 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1a7c5470b472,45799,-1 for getting cluster id 2024-11-23T16:15:41,209 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-23T16:15:41,210 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '34e0592b-548a-4d73-be6a-df1bafadfd92' 2024-11-23T16:15:41,210 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-23T16:15:41,210 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "34e0592b-548a-4d73-be6a-df1bafadfd92" 2024-11-23T16:15:41,210 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1627df26, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T16:15:41,210 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1a7c5470b472,45799,-1] 2024-11-23T16:15:41,210 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-23T16:15:41,210 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T16:15:41,211 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58600, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-23T16:15:41,211 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3af2e762, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T16:15:41,212 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5ae789b8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T16:15:41,212 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1a7c5470b472,45799,-1 for getting cluster id 2024-11-23T16:15:41,212 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-23T16:15:41,212 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '34e0592b-548a-4d73-be6a-df1bafadfd92' 2024-11-23T16:15:41,212 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-23T16:15:41,212 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "34e0592b-548a-4d73-be6a-df1bafadfd92" 2024-11-23T16:15:41,213 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3b883a5a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T16:15:41,213 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1a7c5470b472,45799,-1] 2024-11-23T16:15:41,213 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-23T16:15:41,213 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T16:15:41,213 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58620, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-23T16:15:41,214 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@9b2ab1b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T16:15:41,214 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5b7b3b89, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T16:15:41,214 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1a7c5470b472,45799,-1 for getting cluster id 2024-11-23T16:15:41,214 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-23T16:15:41,215 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '34e0592b-548a-4d73-be6a-df1bafadfd92' 2024-11-23T16:15:41,215 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-23T16:15:41,215 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "34e0592b-548a-4d73-be6a-df1bafadfd92" 2024-11-23T16:15:41,215 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@283a75dc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T16:15:41,215 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1a7c5470b472,45799,-1] 2024-11-23T16:15:41,215 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-23T16:15:41,215 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T16:15:41,220 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58642, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-23T16:15:41,221 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@39e2108a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T16:15:41,221 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7e04f49c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T16:15:41,221 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1a7c5470b472,45799,-1 for getting cluster id 2024-11-23T16:15:41,222 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-23T16:15:41,222 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '34e0592b-548a-4d73-be6a-df1bafadfd92' 2024-11-23T16:15:41,222 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-23T16:15:41,222 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "34e0592b-548a-4d73-be6a-df1bafadfd92" 2024-11-23T16:15:41,223 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@76fe6a79, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T16:15:41,223 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1a7c5470b472,45799,-1] 2024-11-23T16:15:41,223 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-23T16:15:41,223 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T16:15:41,223 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58660, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-23T16:15:41,224 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1d1dc4b4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T16:15:41,224 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32ca199c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T16:15:41,224 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1a7c5470b472,45799,-1 for getting cluster id 2024-11-23T16:15:41,224 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-23T16:15:41,225 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '34e0592b-548a-4d73-be6a-df1bafadfd92' 2024-11-23T16:15:41,225 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-23T16:15:41,225 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "34e0592b-548a-4d73-be6a-df1bafadfd92" 2024-11-23T16:15:41,225 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@643c44e3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T16:15:41,225 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1a7c5470b472,45799,-1] 2024-11-23T16:15:41,225 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-23T16:15:41,225 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T16:15:41,226 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58692, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-23T16:15:41,226 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@363a705, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T16:15:41,227 DEBUG [Thread-1896 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T16:15:41,227 DEBUG [Thread-1883 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T16:15:41,227 DEBUG [Thread-1892 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T16:15:41,227 DEBUG [Thread-1881 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T16:15:41,227 DEBUG [Thread-1900 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T16:15:41,227 DEBUG [Thread-1887 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T16:15:41,228 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-23T16:15:41,228 DEBUG [Thread-1889 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T16:15:41,228 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1a7c5470b472,44715,1732378420375, seqNum=-1] 2024-11-23T16:15:41,228 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1a7c5470b472,44715,1732378420375, seqNum=-1] 2024-11-23T16:15:41,228 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T16:15:41,228 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1a7c5470b472,44715,1732378420375, seqNum=-1] 2024-11-23T16:15:41,228 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T16:15:41,228 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1a7c5470b472,44715,1732378420375, seqNum=-1] 2024-11-23T16:15:41,228 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T16:15:41,229 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T16:15:41,229 DEBUG [Thread-1898 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T16:15:41,229 DEBUG [Thread-1885 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T16:15:41,229 DEBUG [Thread-1894 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T16:15:41,229 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1a7c5470b472,44715,1732378420375, seqNum=-1] 2024-11-23T16:15:41,229 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1a7c5470b472,44715,1732378420375, seqNum=-1] 2024-11-23T16:15:41,229 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1a7c5470b472,44715,1732378420375, seqNum=-1] 2024-11-23T16:15:41,229 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T16:15:41,230 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T16:15:41,230 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39712, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T16:15:41,230 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T16:15:41,230 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39718, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T16:15:41,230 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39720, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T16:15:41,230 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39740, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T16:15:41,230 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1a7c5470b472,44715,1732378420375, seqNum=-1] 2024-11-23T16:15:41,230 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1a7c5470b472,44715,1732378420375, seqNum=-1] 2024-11-23T16:15:41,230 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T16:15:41,230 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T16:15:41,231 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39752, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T16:15:41,231 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39764, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T16:15:41,231 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39780, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T16:15:41,232 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1a7c5470b472,44715,1732378420375, seqNum=-1] 2024-11-23T16:15:41,232 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39754, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T16:15:41,232 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T16:15:41,232 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39788, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T16:15:41,232 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2] 2024-11-23T16:15:41,232 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2] 2024-11-23T16:15:41,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] procedure2.ProcedureExecutor(1139): Stored pid=113, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees 2024-11-23T16:15:41,233 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39792, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T16:15:41,234 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2] 2024-11-23T16:15:41,234 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2] 2024-11-23T16:15:41,234 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2] 2024-11-23T16:15:41,234 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2] 2024-11-23T16:15:41,235 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2] 2024-11-23T16:15:41,236 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2] 2024-11-23T16:15:41,236 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=113, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-23T16:15:41,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-23T16:15:41,237 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2] 2024-11-23T16:15:41,237 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2] 2024-11-23T16:15:41,239 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=113, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-23T16:15:41,239 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=114, ppid=113, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-23T16:15:41,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(8855): Flush requested on 63f43c63344ece03d182c42497a352e4 2024-11-23T16:15:41,251 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 63f43c63344ece03d182c42497a352e4 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-23T16:15:41,252 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 63f43c63344ece03d182c42497a352e4, store=A 2024-11-23T16:15:41,252 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-23T16:15:41,252 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 63f43c63344ece03d182c42497a352e4, store=B 2024-11-23T16:15:41,252 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-23T16:15:41,252 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 63f43c63344ece03d182c42497a352e4, store=C 2024-11-23T16:15:41,252 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-23T16:15:41,277 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/.tmp/A/e407c533aeae46619bebcd49c9222c75 is 50, key is test_row_0/A:col10/1732378541245/Put/seqid=0 2024-11-23T16:15:41,285 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742241_1417 (size=14341) 2024-11-23T16:15:41,294 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:41,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39720 deadline: 1732378601282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:41,295 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:41,295 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:41,295 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:41,295 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:15:41,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39764 deadline: 1732378601287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:41,295 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:41,295 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:41,295 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:15:41,298 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:41,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39740 deadline: 1732378601293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:41,299 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:41,299 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:41,299 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:15:41,302 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:41,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39752 deadline: 1732378601293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:41,302 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:41,302 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:41,302 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:15:41,306 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:41,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39788 deadline: 1732378601295, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:41,307 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:41,307 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:41,307 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:15:41,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-23T16:15:41,399 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-23T16:15:41,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4. 2024-11-23T16:15:41,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4. as already flushing 2024-11-23T16:15:41,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4. 2024-11-23T16:15:41,399 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 63f43c63344ece03d182c42497a352e4, NAME => 'TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:15:41,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 63f43c63344ece03d182c42497a352e4, NAME => 'TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:15:41,400 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:41,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 63f43c63344ece03d182c42497a352e4, NAME => 'TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 63f43c63344ece03d182c42497a352e4, NAME => 'TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:15:41,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39720 deadline: 1732378601398, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:41,400 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:41,400 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:41,400 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:41,400 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:15:41,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39764 deadline: 1732378601398, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:41,401 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:41,401 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:41,401 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:15:41,412 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:41,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39740 deadline: 1732378601407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:41,412 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:41,413 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:41,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39752 deadline: 1732378601407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:41,413 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:41,413 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:15:41,413 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:41,413 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:41,413 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:15:41,425 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:41,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39788 deadline: 1732378601418, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:41,425 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:41,425 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:41,425 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:15:41,552 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-23T16:15:41,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4. 2024-11-23T16:15:41,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4. as already flushing 2024-11-23T16:15:41,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4. 2024-11-23T16:15:41,552 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 63f43c63344ece03d182c42497a352e4, NAME => 'TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:15:41,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 63f43c63344ece03d182c42497a352e4, NAME => 'TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:15:41,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 63f43c63344ece03d182c42497a352e4, NAME => 'TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 63f43c63344ece03d182c42497a352e4, NAME => 'TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:15:41,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-23T16:15:41,619 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:41,620 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:41,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39720 deadline: 1732378601607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:41,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39764 deadline: 1732378601607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:41,620 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:41,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39740 deadline: 1732378601618, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:41,620 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:41,620 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:41,620 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:41,620 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:41,620 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:15:41,620 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:15:41,620 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:41,620 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:41,621 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:41,621 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:15:41,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39752 deadline: 1732378601618, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:41,621 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:41,621 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:41,621 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:15:41,635 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:41,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39788 deadline: 1732378601628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:41,635 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:41,635 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:41,635 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:15:41,686 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/.tmp/A/e407c533aeae46619bebcd49c9222c75 2024-11-23T16:15:41,705 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-23T16:15:41,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4. 2024-11-23T16:15:41,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4. as already flushing 2024-11-23T16:15:41,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4. 2024-11-23T16:15:41,705 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 63f43c63344ece03d182c42497a352e4, NAME => 'TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:15:41,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 63f43c63344ece03d182c42497a352e4, NAME => 'TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:15:41,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 63f43c63344ece03d182c42497a352e4, NAME => 'TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 63f43c63344ece03d182c42497a352e4, NAME => 'TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:15:41,710 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/.tmp/B/c77e2cf5829b48a18e01b5ee7112880b is 50, key is test_row_0/B:col10/1732378541245/Put/seqid=0 2024-11-23T16:15:41,716 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742242_1418 (size=12001) 2024-11-23T16:15:41,858 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-23T16:15:41,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4. 2024-11-23T16:15:41,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4. as already flushing 2024-11-23T16:15:41,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4. 2024-11-23T16:15:41,858 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 63f43c63344ece03d182c42497a352e4, NAME => 'TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:15:41,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 63f43c63344ece03d182c42497a352e4, NAME => 'TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:15:41,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 63f43c63344ece03d182c42497a352e4, NAME => 'TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 63f43c63344ece03d182c42497a352e4, NAME => 'TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:15:41,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-23T16:15:41,934 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:41,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39764 deadline: 1732378601928, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:41,935 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:41,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39720 deadline: 1732378601928, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:41,935 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:41,935 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:41,935 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:15:41,935 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:41,935 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:41,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39740 deadline: 1732378601928, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:41,935 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:41,935 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:15:41,935 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:41,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39752 deadline: 1732378601928, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:41,935 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:41,935 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:41,935 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:15:41,935 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:41,935 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:41,936 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:15:41,946 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:41,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39788 deadline: 1732378601938, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:41,947 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:41,947 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:41,947 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:15:42,011 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-23T16:15:42,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4. 2024-11-23T16:15:42,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4. as already flushing 2024-11-23T16:15:42,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4. 2024-11-23T16:15:42,011 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 63f43c63344ece03d182c42497a352e4, NAME => 'TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:15:42,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 63f43c63344ece03d182c42497a352e4, NAME => 'TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:15:42,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 63f43c63344ece03d182c42497a352e4, NAME => 'TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 63f43c63344ece03d182c42497a352e4, NAME => 'TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:15:42,116 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/.tmp/B/c77e2cf5829b48a18e01b5ee7112880b 2024-11-23T16:15:42,136 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/.tmp/C/b50cebd05b6a43169e0d7a0e730aa3b9 is 50, key is test_row_0/C:col10/1732378541245/Put/seqid=0 2024-11-23T16:15:42,140 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742243_1419 (size=12001) 2024-11-23T16:15:42,140 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/.tmp/C/b50cebd05b6a43169e0d7a0e730aa3b9 2024-11-23T16:15:42,143 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/.tmp/A/e407c533aeae46619bebcd49c9222c75 as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/A/e407c533aeae46619bebcd49c9222c75 2024-11-23T16:15:42,147 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/A/e407c533aeae46619bebcd49c9222c75, entries=200, sequenceid=13, filesize=14.0 K 2024-11-23T16:15:42,148 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/.tmp/B/c77e2cf5829b48a18e01b5ee7112880b as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/B/c77e2cf5829b48a18e01b5ee7112880b 2024-11-23T16:15:42,154 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/B/c77e2cf5829b48a18e01b5ee7112880b, entries=150, sequenceid=13, filesize=11.7 K 2024-11-23T16:15:42,155 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/.tmp/C/b50cebd05b6a43169e0d7a0e730aa3b9 as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/C/b50cebd05b6a43169e0d7a0e730aa3b9 2024-11-23T16:15:42,159 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/C/b50cebd05b6a43169e0d7a0e730aa3b9, entries=150, sequenceid=13, filesize=11.7 K 2024-11-23T16:15:42,160 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 63f43c63344ece03d182c42497a352e4 in 909ms, sequenceid=13, compaction requested=false 2024-11-23T16:15:42,160 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-23T16:15:42,161 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 63f43c63344ece03d182c42497a352e4: 2024-11-23T16:15:42,163 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-23T16:15:42,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4. 2024-11-23T16:15:42,164 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2902): Flushing 63f43c63344ece03d182c42497a352e4 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-23T16:15:42,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 63f43c63344ece03d182c42497a352e4, store=A 2024-11-23T16:15:42,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-23T16:15:42,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 63f43c63344ece03d182c42497a352e4, store=B 2024-11-23T16:15:42,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-23T16:15:42,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 63f43c63344ece03d182c42497a352e4, store=C 2024-11-23T16:15:42,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-23T16:15:42,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/.tmp/A/1ae48cc8fc0b4836b60eddf3acb17fb5 is 50, key is test_row_0/A:col10/1732378541291/Put/seqid=0 2024-11-23T16:15:42,202 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742244_1420 (size=12001) 2024-11-23T16:15:42,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-23T16:15:42,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(8855): Flush requested on 63f43c63344ece03d182c42497a352e4 2024-11-23T16:15:42,442 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4. as already flushing 2024-11-23T16:15:42,469 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:42,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39788 deadline: 1732378602458, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:42,470 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:42,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39752 deadline: 1732378602459, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:42,470 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:42,470 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:42,470 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:15:42,470 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:42,470 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:42,470 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:42,470 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:15:42,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39720 deadline: 1732378602460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:42,470 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:42,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39764 deadline: 1732378602461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:42,471 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:42,471 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:42,471 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:15:42,471 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:42,471 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:42,471 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:15:42,472 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:42,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39740 deadline: 1732378602470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:42,473 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:42,473 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:42,473 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:15:42,585 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:42,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39752 deadline: 1732378602578, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:42,585 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:42,585 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:42,585 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:42,585 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:15:42,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39720 deadline: 1732378602578, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:42,586 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:42,586 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:42,586 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:42,586 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:15:42,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39764 deadline: 1732378602578, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:42,586 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:42,586 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:42,586 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:42,586 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:15:42,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39740 deadline: 1732378602579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:42,586 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:42,586 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:42,586 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:15:42,603 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/.tmp/A/1ae48cc8fc0b4836b60eddf3acb17fb5 2024-11-23T16:15:42,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/.tmp/B/9795dabb27ea4c568cf1ef52fad9bf4f is 50, key is test_row_0/B:col10/1732378541291/Put/seqid=0 2024-11-23T16:15:42,615 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742245_1421 (size=12001) 2024-11-23T16:15:42,775 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-23T16:15:42,794 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:42,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39752 deadline: 1732378602788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:42,794 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:42,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39720 deadline: 1732378602788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:42,795 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:42,795 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:42,795 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:15:42,796 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:42,796 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:42,796 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:15:42,810 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:42,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39764 deadline: 1732378602798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:42,810 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:42,810 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:42,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39740 deadline: 1732378602798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:42,811 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:42,811 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:15:42,811 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:42,811 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:42,811 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:15:43,015 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/.tmp/B/9795dabb27ea4c568cf1ef52fad9bf4f 2024-11-23T16:15:43,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/.tmp/C/a50a1449830142ef80ec28c72a09b387 is 50, key is test_row_0/C:col10/1732378541291/Put/seqid=0 2024-11-23T16:15:43,026 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742246_1422 (size=12001) 2024-11-23T16:15:43,119 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:43,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39752 deadline: 1732378603108, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:43,119 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:43,119 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:43,119 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:43,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39720 deadline: 1732378603108, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:43,119 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:15:43,119 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:43,119 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:43,120 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:15:43,128 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:43,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39764 deadline: 1732378603118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:43,128 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:43,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39740 deadline: 1732378603118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:43,128 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:43,128 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:43,128 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:15:43,128 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:43,128 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:43,129 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:15:43,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-23T16:15:43,427 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/.tmp/C/a50a1449830142ef80ec28c72a09b387 2024-11-23T16:15:43,431 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/.tmp/A/1ae48cc8fc0b4836b60eddf3acb17fb5 as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/A/1ae48cc8fc0b4836b60eddf3acb17fb5 2024-11-23T16:15:43,434 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/A/1ae48cc8fc0b4836b60eddf3acb17fb5, entries=150, sequenceid=33, filesize=11.7 K 2024-11-23T16:15:43,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/.tmp/B/9795dabb27ea4c568cf1ef52fad9bf4f as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/B/9795dabb27ea4c568cf1ef52fad9bf4f 2024-11-23T16:15:43,438 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/B/9795dabb27ea4c568cf1ef52fad9bf4f, entries=150, sequenceid=33, filesize=11.7 K 2024-11-23T16:15:43,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/.tmp/C/a50a1449830142ef80ec28c72a09b387 as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/C/a50a1449830142ef80ec28c72a09b387 2024-11-23T16:15:43,442 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/C/a50a1449830142ef80ec28c72a09b387, entries=150, sequenceid=33, filesize=11.7 K 2024-11-23T16:15:43,443 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 63f43c63344ece03d182c42497a352e4 in 1279ms, sequenceid=33, compaction requested=false 2024-11-23T16:15:43,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2603): Flush status journal for 63f43c63344ece03d182c42497a352e4: 2024-11-23T16:15:43,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4. 2024-11-23T16:15:43,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=114 2024-11-23T16:15:43,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.HMaster(4169): Remote procedure done, pid=114 2024-11-23T16:15:43,446 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=114, resume processing ppid=113 2024-11-23T16:15:43,446 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=114, ppid=113, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2050 sec 2024-11-23T16:15:43,448 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=113, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees in 2.2190 sec 2024-11-23T16:15:43,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(8855): Flush requested on 63f43c63344ece03d182c42497a352e4 2024-11-23T16:15:43,482 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 63f43c63344ece03d182c42497a352e4 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-23T16:15:43,482 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 63f43c63344ece03d182c42497a352e4, store=A 2024-11-23T16:15:43,482 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-23T16:15:43,483 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 63f43c63344ece03d182c42497a352e4, store=B 2024-11-23T16:15:43,483 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-23T16:15:43,483 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 63f43c63344ece03d182c42497a352e4, store=C 2024-11-23T16:15:43,483 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-23T16:15:43,487 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/.tmp/A/c7eb4f5aa1c746b9af5c69ac0701fa0c is 50, key is test_row_0/A:col10/1732378542458/Put/seqid=0 2024-11-23T16:15:43,490 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742247_1423 (size=14341) 2024-11-23T16:15:43,607 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:43,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39788 deadline: 1732378603598, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:43,608 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:43,608 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:43,608 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:15:43,634 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:43,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39752 deadline: 1732378603628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:43,634 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:43,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39720 deadline: 1732378603628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:43,635 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:43,635 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:43,635 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:43,635 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:43,635 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:15:43,635 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:15:43,647 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:43,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39764 deadline: 1732378603638, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:43,647 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:43,648 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:43,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39740 deadline: 1732378603638, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:43,648 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:43,648 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:15:43,648 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:43,648 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:43,648 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:15:43,722 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:43,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39788 deadline: 1732378603718, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:43,722 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:43,722 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:43,722 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:15:43,891 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=46 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/.tmp/A/c7eb4f5aa1c746b9af5c69ac0701fa0c 2024-11-23T16:15:43,898 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/.tmp/B/c07aec55d0ff47fdb88fae1987d71c5e is 50, key is test_row_0/B:col10/1732378542458/Put/seqid=0 2024-11-23T16:15:43,901 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742248_1424 (size=12001) 2024-11-23T16:15:43,902 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=46 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/.tmp/B/c07aec55d0ff47fdb88fae1987d71c5e 2024-11-23T16:15:43,915 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/.tmp/C/2f81d9d5266d4015915d2cbccb677729 is 50, key is test_row_0/C:col10/1732378542458/Put/seqid=0 2024-11-23T16:15:43,922 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742249_1425 (size=12001) 2024-11-23T16:15:43,932 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:43,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39788 deadline: 1732378603928, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:43,932 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:43,933 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:43,933 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:15:44,248 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:44,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39788 deadline: 1732378604238, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:44,249 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:44,249 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:44,249 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:15:44,322 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=46 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/.tmp/C/2f81d9d5266d4015915d2cbccb677729 2024-11-23T16:15:44,326 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/.tmp/A/c7eb4f5aa1c746b9af5c69ac0701fa0c as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/A/c7eb4f5aa1c746b9af5c69ac0701fa0c 2024-11-23T16:15:44,330 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/A/c7eb4f5aa1c746b9af5c69ac0701fa0c, entries=200, sequenceid=46, filesize=14.0 K 2024-11-23T16:15:44,331 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/.tmp/B/c07aec55d0ff47fdb88fae1987d71c5e as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/B/c07aec55d0ff47fdb88fae1987d71c5e 2024-11-23T16:15:44,334 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/B/c07aec55d0ff47fdb88fae1987d71c5e, entries=150, sequenceid=46, filesize=11.7 K 2024-11-23T16:15:44,335 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/.tmp/C/2f81d9d5266d4015915d2cbccb677729 as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/C/2f81d9d5266d4015915d2cbccb677729 2024-11-23T16:15:44,338 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/C/2f81d9d5266d4015915d2cbccb677729, entries=150, sequenceid=46, filesize=11.7 K 2024-11-23T16:15:44,339 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 63f43c63344ece03d182c42497a352e4 in 857ms, sequenceid=46, compaction requested=true 2024-11-23T16:15:44,339 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 63f43c63344ece03d182c42497a352e4: 2024-11-23T16:15:44,339 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 63f43c63344ece03d182c42497a352e4:A, priority=-2147483648, current under compaction store size is 1 2024-11-23T16:15:44,339 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T16:15:44,339 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 63f43c63344ece03d182c42497a352e4:B, priority=-2147483648, current under compaction store size is 2 2024-11-23T16:15:44,339 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T16:15:44,339 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 63f43c63344ece03d182c42497a352e4:C, priority=-2147483648, current under compaction store size is 3 2024-11-23T16:15:44,339 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T16:15:44,339 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T16:15:44,339 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T16:15:44,340 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T16:15:44,340 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40683 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T16:15:44,340 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HStore(1541): 63f43c63344ece03d182c42497a352e4/A is initiating minor compaction (all files) 2024-11-23T16:15:44,340 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HStore(1541): 63f43c63344ece03d182c42497a352e4/B is initiating minor compaction (all files) 2024-11-23T16:15:44,340 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 63f43c63344ece03d182c42497a352e4/A in TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4. 2024-11-23T16:15:44,340 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 63f43c63344ece03d182c42497a352e4/B in TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4. 2024-11-23T16:15:44,340 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/B/c77e2cf5829b48a18e01b5ee7112880b, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/B/9795dabb27ea4c568cf1ef52fad9bf4f, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/B/c07aec55d0ff47fdb88fae1987d71c5e] into tmpdir=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/.tmp, totalSize=35.2 K 2024-11-23T16:15:44,340 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/A/e407c533aeae46619bebcd49c9222c75, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/A/1ae48cc8fc0b4836b60eddf3acb17fb5, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/A/c7eb4f5aa1c746b9af5c69ac0701fa0c] into tmpdir=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/.tmp, totalSize=39.7 K 2024-11-23T16:15:44,340 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.Compactor(225): Compacting c77e2cf5829b48a18e01b5ee7112880b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1732378541245 2024-11-23T16:15:44,340 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.Compactor(225): Compacting e407c533aeae46619bebcd49c9222c75, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1732378541245 2024-11-23T16:15:44,341 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.Compactor(225): Compacting 9795dabb27ea4c568cf1ef52fad9bf4f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=33, earliestPutTs=1732378541291 2024-11-23T16:15:44,341 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1ae48cc8fc0b4836b60eddf3acb17fb5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=33, earliestPutTs=1732378541291 2024-11-23T16:15:44,341 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.Compactor(225): Compacting c07aec55d0ff47fdb88fae1987d71c5e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1732378542458 2024-11-23T16:15:44,341 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.Compactor(225): Compacting c7eb4f5aa1c746b9af5c69ac0701fa0c, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1732378542458 2024-11-23T16:15:44,347 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 63f43c63344ece03d182c42497a352e4#B#compaction#351 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-23T16:15:44,347 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/.tmp/B/df4916605cc1401c9bd809a39c46dec9 is 50, key is test_row_0/B:col10/1732378542458/Put/seqid=0 2024-11-23T16:15:44,348 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 63f43c63344ece03d182c42497a352e4#A#compaction#352 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T16:15:44,348 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/.tmp/A/ff1641249c5d4a988529f2e1773cd2d2 is 50, key is test_row_0/A:col10/1732378542458/Put/seqid=0 2024-11-23T16:15:44,354 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742250_1426 (size=12104) 2024-11-23T16:15:44,366 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742251_1427 (size=12104) 2024-11-23T16:15:44,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(8855): Flush requested on 63f43c63344ece03d182c42497a352e4 2024-11-23T16:15:44,653 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 63f43c63344ece03d182c42497a352e4 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-23T16:15:44,653 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 63f43c63344ece03d182c42497a352e4, store=A 2024-11-23T16:15:44,653 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-23T16:15:44,653 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 63f43c63344ece03d182c42497a352e4, store=B 2024-11-23T16:15:44,653 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-23T16:15:44,653 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 63f43c63344ece03d182c42497a352e4, store=C 2024-11-23T16:15:44,653 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-23T16:15:44,657 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/.tmp/A/e2101deb540c4868823ae0c48251af1e is 50, key is test_row_0/A:col10/1732378543523/Put/seqid=0 2024-11-23T16:15:44,661 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742252_1428 (size=14341) 2024-11-23T16:15:44,662 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/.tmp/A/e2101deb540c4868823ae0c48251af1e 2024-11-23T16:15:44,668 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/.tmp/B/8a625a77afa2499d970a7d13f7c288ba is 50, key is test_row_0/B:col10/1732378543523/Put/seqid=0 2024-11-23T16:15:44,671 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742253_1429 (size=12001) 2024-11-23T16:15:44,672 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/.tmp/B/8a625a77afa2499d970a7d13f7c288ba 2024-11-23T16:15:44,679 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/.tmp/C/7ca85a5b077544a2aea19ab716007e42 is 50, key is test_row_0/C:col10/1732378543523/Put/seqid=0 2024-11-23T16:15:44,682 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742254_1430 (size=12001) 2024-11-23T16:15:44,682 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/.tmp/C/7ca85a5b077544a2aea19ab716007e42 2024-11-23T16:15:44,686 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:44,686 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/.tmp/A/e2101deb540c4868823ae0c48251af1e as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/A/e2101deb540c4868823ae0c48251af1e 2024-11-23T16:15:44,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39752 deadline: 1732378604672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:44,687 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:44,687 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:44,687 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:15:44,688 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:44,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39720 deadline: 1732378604673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:44,690 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:44,690 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:44,690 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:15:44,690 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/A/e2101deb540c4868823ae0c48251af1e, entries=200, sequenceid=66, filesize=14.0 K 2024-11-23T16:15:44,691 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/.tmp/B/8a625a77afa2499d970a7d13f7c288ba as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/B/8a625a77afa2499d970a7d13f7c288ba 2024-11-23T16:15:44,693 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:44,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39764 deadline: 1732378604685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:44,694 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:44,694 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:44,694 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:15:44,695 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/B/8a625a77afa2499d970a7d13f7c288ba, entries=150, sequenceid=66, filesize=11.7 K 2024-11-23T16:15:44,695 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/.tmp/C/7ca85a5b077544a2aea19ab716007e42 as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/C/7ca85a5b077544a2aea19ab716007e42 2024-11-23T16:15:44,698 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/C/7ca85a5b077544a2aea19ab716007e42, entries=150, sequenceid=66, filesize=11.7 K 2024-11-23T16:15:44,699 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 63f43c63344ece03d182c42497a352e4 in 47ms, sequenceid=66, compaction requested=true 2024-11-23T16:15:44,699 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 63f43c63344ece03d182c42497a352e4: 2024-11-23T16:15:44,699 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 63f43c63344ece03d182c42497a352e4:A, priority=-2147483648, current under compaction store size is 3 2024-11-23T16:15:44,699 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-23T16:15:44,699 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 63f43c63344ece03d182c42497a352e4:B, priority=-2147483648, current under compaction store size is 3 2024-11-23T16:15:44,699 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-11-23T16:15:44,699 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 63f43c63344ece03d182c42497a352e4:C, priority=-2147483648, current under compaction store size is 3 2024-11-23T16:15:44,699 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=4), splitQueue=0 2024-11-23T16:15:44,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(8855): Flush requested on 63f43c63344ece03d182c42497a352e4 2024-11-23T16:15:44,702 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 63f43c63344ece03d182c42497a352e4 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-23T16:15:44,702 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 63f43c63344ece03d182c42497a352e4, store=A 2024-11-23T16:15:44,702 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-23T16:15:44,702 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 63f43c63344ece03d182c42497a352e4, store=B 2024-11-23T16:15:44,702 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-23T16:15:44,702 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 63f43c63344ece03d182c42497a352e4, store=C 2024-11-23T16:15:44,702 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-23T16:15:44,707 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/.tmp/A/49462c0916b149179a8e28b17cb17d66 is 50, key is test_row_0/A:col10/1732378544667/Put/seqid=0 2024-11-23T16:15:44,712 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742255_1431 (size=14341) 2024-11-23T16:15:44,712 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=79 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/.tmp/A/49462c0916b149179a8e28b17cb17d66 2024-11-23T16:15:44,721 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/.tmp/B/eb993843f605415ca294ce328eb2875c is 50, key is test_row_0/B:col10/1732378544667/Put/seqid=0 2024-11-23T16:15:44,723 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742256_1432 (size=12001) 2024-11-23T16:15:44,725 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=79 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/.tmp/B/eb993843f605415ca294ce328eb2875c 2024-11-23T16:15:44,731 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/.tmp/C/c4d416dc87864934a5bda95181df5631 is 50, key is test_row_0/C:col10/1732378544667/Put/seqid=0 2024-11-23T16:15:44,734 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742257_1433 (size=12001) 2024-11-23T16:15:44,736 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=79 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/.tmp/C/c4d416dc87864934a5bda95181df5631 2024-11-23T16:15:44,739 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/.tmp/A/49462c0916b149179a8e28b17cb17d66 as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/A/49462c0916b149179a8e28b17cb17d66 2024-11-23T16:15:44,742 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/A/49462c0916b149179a8e28b17cb17d66, entries=200, sequenceid=79, filesize=14.0 K 2024-11-23T16:15:44,743 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/.tmp/B/eb993843f605415ca294ce328eb2875c as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/B/eb993843f605415ca294ce328eb2875c 2024-11-23T16:15:44,746 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/B/eb993843f605415ca294ce328eb2875c, entries=150, sequenceid=79, filesize=11.7 K 2024-11-23T16:15:44,746 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/.tmp/C/c4d416dc87864934a5bda95181df5631 as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/C/c4d416dc87864934a5bda95181df5631 2024-11-23T16:15:44,750 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/C/c4d416dc87864934a5bda95181df5631, entries=150, sequenceid=79, filesize=11.7 K 2024-11-23T16:15:44,751 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=40.25 KB/41220 for 63f43c63344ece03d182c42497a352e4 in 49ms, sequenceid=79, compaction requested=true 2024-11-23T16:15:44,751 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 63f43c63344ece03d182c42497a352e4: 2024-11-23T16:15:44,751 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 63f43c63344ece03d182c42497a352e4:A, priority=-2147483648, current under compaction store size is 3 2024-11-23T16:15:44,751 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=5), splitQueue=0 2024-11-23T16:15:44,751 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 63f43c63344ece03d182c42497a352e4:B, priority=-2147483648, current under compaction store size is 3 2024-11-23T16:15:44,751 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=6), splitQueue=0 2024-11-23T16:15:44,751 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 63f43c63344ece03d182c42497a352e4:C, priority=-2147483648, current under compaction store size is 3 2024-11-23T16:15:44,751 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=7), splitQueue=0 2024-11-23T16:15:44,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(8855): Flush requested on 63f43c63344ece03d182c42497a352e4 2024-11-23T16:15:44,765 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 63f43c63344ece03d182c42497a352e4 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-23T16:15:44,765 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 63f43c63344ece03d182c42497a352e4, store=A 2024-11-23T16:15:44,765 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-23T16:15:44,766 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 63f43c63344ece03d182c42497a352e4, store=B 2024-11-23T16:15:44,766 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-23T16:15:44,766 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 63f43c63344ece03d182c42497a352e4, store=C 2024-11-23T16:15:44,766 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-23T16:15:44,766 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/.tmp/B/df4916605cc1401c9bd809a39c46dec9 as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/B/df4916605cc1401c9bd809a39c46dec9 2024-11-23T16:15:44,773 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/.tmp/A/83acfcade2b6405e88a009b9937c1f29 is 50, key is test_row_0/A:col10/1732378544733/Put/seqid=0 2024-11-23T16:15:44,776 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 63f43c63344ece03d182c42497a352e4/B of 63f43c63344ece03d182c42497a352e4 into df4916605cc1401c9bd809a39c46dec9(size=11.8 K), total size for store is 35.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T16:15:44,776 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 63f43c63344ece03d182c42497a352e4: 2024-11-23T16:15:44,776 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., storeName=63f43c63344ece03d182c42497a352e4/B, priority=13, startTime=1732378544339; duration=0sec 2024-11-23T16:15:44,776 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=7), splitQueue=0 2024-11-23T16:15:44,776 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 63f43c63344ece03d182c42497a352e4:B 2024-11-23T16:15:44,776 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 3 compacting, 2 eligible, 16 blocking 2024-11-23T16:15:44,779 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-23T16:15:44,779 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-23T16:15:44,779 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4. because compaction request was cancelled 2024-11-23T16:15:44,779 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 63f43c63344ece03d182c42497a352e4:A 2024-11-23T16:15:44,779 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-23T16:15:44,780 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/.tmp/A/ff1641249c5d4a988529f2e1773cd2d2 as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/A/ff1641249c5d4a988529f2e1773cd2d2 2024-11-23T16:15:44,781 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 60005 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-23T16:15:44,781 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HStore(1541): 63f43c63344ece03d182c42497a352e4/C is initiating minor compaction (all files) 2024-11-23T16:15:44,781 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 63f43c63344ece03d182c42497a352e4/C in TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4. 2024-11-23T16:15:44,781 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/C/b50cebd05b6a43169e0d7a0e730aa3b9, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/C/a50a1449830142ef80ec28c72a09b387, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/C/2f81d9d5266d4015915d2cbccb677729, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/C/7ca85a5b077544a2aea19ab716007e42, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/C/c4d416dc87864934a5bda95181df5631] into tmpdir=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/.tmp, totalSize=58.6 K 2024-11-23T16:15:44,782 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.Compactor(225): Compacting b50cebd05b6a43169e0d7a0e730aa3b9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1732378541245 2024-11-23T16:15:44,782 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.Compactor(225): Compacting a50a1449830142ef80ec28c72a09b387, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=33, earliestPutTs=1732378541291 2024-11-23T16:15:44,782 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.Compactor(225): Compacting 2f81d9d5266d4015915d2cbccb677729, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1732378542458 2024-11-23T16:15:44,782 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.Compactor(225): Compacting 7ca85a5b077544a2aea19ab716007e42, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=66, earliestPutTs=1732378543523 2024-11-23T16:15:44,785 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.Compactor(225): Compacting c4d416dc87864934a5bda95181df5631, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1732378544667 2024-11-23T16:15:44,786 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 63f43c63344ece03d182c42497a352e4/A of 63f43c63344ece03d182c42497a352e4 into ff1641249c5d4a988529f2e1773cd2d2(size=11.8 K), total size for store is 39.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T16:15:44,786 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 63f43c63344ece03d182c42497a352e4: 2024-11-23T16:15:44,786 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., storeName=63f43c63344ece03d182c42497a352e4/A, priority=13, startTime=1732378544339; duration=0sec 2024-11-23T16:15:44,786 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=5), splitQueue=0 2024-11-23T16:15:44,786 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 63f43c63344ece03d182c42497a352e4:A 2024-11-23T16:15:44,786 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 63f43c63344ece03d182c42497a352e4:B 2024-11-23T16:15:44,786 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 5 compacting, 0 eligible, 16 blocking 2024-11-23T16:15:44,786 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-23T16:15:44,786 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-23T16:15:44,786 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4. because compaction request was cancelled 2024-11-23T16:15:44,786 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 63f43c63344ece03d182c42497a352e4:C 2024-11-23T16:15:44,786 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 63f43c63344ece03d182c42497a352e4:A 2024-11-23T16:15:44,786 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 63f43c63344ece03d182c42497a352e4:B 2024-11-23T16:15:44,786 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742258_1434 (size=14341) 2024-11-23T16:15:44,786 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T16:15:44,787 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40786 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T16:15:44,787 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HStore(1541): 63f43c63344ece03d182c42497a352e4/A is initiating minor compaction (all files) 2024-11-23T16:15:44,787 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 63f43c63344ece03d182c42497a352e4/A in TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4. 2024-11-23T16:15:44,787 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/A/ff1641249c5d4a988529f2e1773cd2d2, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/A/e2101deb540c4868823ae0c48251af1e, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/A/49462c0916b149179a8e28b17cb17d66] into tmpdir=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/.tmp, totalSize=39.8 K 2024-11-23T16:15:44,788 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.Compactor(225): Compacting ff1641249c5d4a988529f2e1773cd2d2, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1732378542458 2024-11-23T16:15:44,788 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.Compactor(225): Compacting e2101deb540c4868823ae0c48251af1e, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=66, earliestPutTs=1732378543523 2024-11-23T16:15:44,788 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.Compactor(225): Compacting 49462c0916b149179a8e28b17cb17d66, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1732378544667 2024-11-23T16:15:44,796 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 63f43c63344ece03d182c42497a352e4#C#compaction#360 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T16:15:44,796 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/.tmp/C/457649d6dd8b4269824f8f975d3c4c42 is 50, key is test_row_0/C:col10/1732378544667/Put/seqid=0 2024-11-23T16:15:44,800 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 63f43c63344ece03d182c42497a352e4#A#compaction#361 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T16:15:44,800 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/.tmp/A/bcd9e3e7285b4b1f8ce668f410f8b772 is 50, key is test_row_0/A:col10/1732378544667/Put/seqid=0 2024-11-23T16:15:44,803 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742259_1435 (size=12173) 2024-11-23T16:15:44,805 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742260_1436 (size=12207) 2024-11-23T16:15:44,808 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/.tmp/C/457649d6dd8b4269824f8f975d3c4c42 as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/C/457649d6dd8b4269824f8f975d3c4c42 2024-11-23T16:15:44,813 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 63f43c63344ece03d182c42497a352e4/C of 63f43c63344ece03d182c42497a352e4 into 457649d6dd8b4269824f8f975d3c4c42(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T16:15:44,813 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 63f43c63344ece03d182c42497a352e4: 2024-11-23T16:15:44,813 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., storeName=63f43c63344ece03d182c42497a352e4/C, priority=11, startTime=1732378544751; duration=0sec 2024-11-23T16:15:44,813 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-11-23T16:15:44,813 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 63f43c63344ece03d182c42497a352e4:C 2024-11-23T16:15:44,813 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T16:15:44,814 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T16:15:44,814 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HStore(1541): 63f43c63344ece03d182c42497a352e4/B is initiating minor compaction (all files) 2024-11-23T16:15:44,815 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 63f43c63344ece03d182c42497a352e4/B in TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4. 2024-11-23T16:15:44,815 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/B/df4916605cc1401c9bd809a39c46dec9, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/B/8a625a77afa2499d970a7d13f7c288ba, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/B/eb993843f605415ca294ce328eb2875c] into tmpdir=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/.tmp, totalSize=35.3 K 2024-11-23T16:15:44,815 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.Compactor(225): Compacting df4916605cc1401c9bd809a39c46dec9, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1732378542458 2024-11-23T16:15:44,815 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.Compactor(225): Compacting 8a625a77afa2499d970a7d13f7c288ba, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=66, earliestPutTs=1732378543523 2024-11-23T16:15:44,816 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.Compactor(225): Compacting eb993843f605415ca294ce328eb2875c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1732378544667 2024-11-23T16:15:44,831 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 63f43c63344ece03d182c42497a352e4#B#compaction#362 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T16:15:44,831 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/.tmp/B/79f4376771074b9ea9180bc24137a064 is 50, key is test_row_0/B:col10/1732378544667/Put/seqid=0 2024-11-23T16:15:44,837 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742261_1437 (size=12207) 2024-11-23T16:15:44,842 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1a7c5470b472:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1732378544816, max timestamp=1732378544838Num uniques 859; 2024-11-23T16:15:44,842 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1a7c5470b472:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1732378544816, max timestamp=1732378544838Num uniques 859; 2024-11-23T16:15:44,842 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1a7c5470b472:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-11-23T16:15:44,842 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1a7c5470b472:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-11-23T16:15:44,845 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1a7c5470b472:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1732378544816, max timestamp=1732378544838Num uniques 859; 2024-11-23T16:15:44,845 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1a7c5470b472:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-11-23T16:15:44,847 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/.tmp/B/79f4376771074b9ea9180bc24137a064 as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/B/79f4376771074b9ea9180bc24137a064 2024-11-23T16:15:44,854 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 63f43c63344ece03d182c42497a352e4/B of 63f43c63344ece03d182c42497a352e4 into 79f4376771074b9ea9180bc24137a064(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T16:15:44,854 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 63f43c63344ece03d182c42497a352e4: 2024-11-23T16:15:44,854 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., storeName=63f43c63344ece03d182c42497a352e4/B, priority=13, startTime=1732378544699; duration=0sec 2024-11-23T16:15:44,854 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-23T16:15:44,854 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 63f43c63344ece03d182c42497a352e4:B 2024-11-23T16:15:44,854 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 63f43c63344ece03d182c42497a352e4:C 2024-11-23T16:15:44,854 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 63f43c63344ece03d182c42497a352e4:B 2024-11-23T16:15:44,854 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 1 store files, 0 compacting, 1 eligible, 16 blocking 2024-11-23T16:15:44,855 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-23T16:15:44,855 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-23T16:15:44,855 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4. because compaction request was cancelled 2024-11-23T16:15:44,855 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 63f43c63344ece03d182c42497a352e4:C 2024-11-23T16:15:44,855 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 1 store files, 0 compacting, 1 eligible, 16 blocking 2024-11-23T16:15:44,855 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-23T16:15:44,855 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-23T16:15:44,855 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4. because compaction request was cancelled 2024-11-23T16:15:44,856 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 63f43c63344ece03d182c42497a352e4:B 2024-11-23T16:15:44,891 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:44,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39788 deadline: 1732378604877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:44,892 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:44,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39740 deadline: 1732378604877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:44,892 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:44,892 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:44,892 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:15:44,892 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:44,892 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:44,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39720 deadline: 1732378604878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:44,892 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:44,892 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:15:44,893 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:44,893 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:44,893 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:15:44,900 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:44,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39764 deadline: 1732378604882, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:44,901 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:44,901 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:44,901 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:44,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39752 deadline: 1732378604888, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:44,901 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:15:44,901 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:44,901 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:44,901 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:15:45,010 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:45,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39788 deadline: 1732378604998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:45,010 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:45,011 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:45,011 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:45,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39740 deadline: 1732378604998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:45,011 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:15:45,011 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:45,011 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:45,011 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:15:45,011 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:45,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39720 deadline: 1732378604998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:45,011 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:45,011 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:45,011 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:15:45,017 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:45,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39764 deadline: 1732378605008, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:45,017 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:45,017 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:45,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39752 deadline: 1732378605008, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:45,017 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:45,017 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:15:45,018 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:45,018 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:45,018 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:15:45,187 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=89 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/.tmp/A/83acfcade2b6405e88a009b9937c1f29 2024-11-23T16:15:45,195 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/.tmp/B/e6778eae86ab40e7bbb9e5e24c3abf08 is 50, key is test_row_0/B:col10/1732378544733/Put/seqid=0 2024-11-23T16:15:45,198 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742262_1438 (size=12001) 2024-11-23T16:15:45,209 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/.tmp/A/bcd9e3e7285b4b1f8ce668f410f8b772 as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/A/bcd9e3e7285b4b1f8ce668f410f8b772 2024-11-23T16:15:45,213 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 63f43c63344ece03d182c42497a352e4/A of 63f43c63344ece03d182c42497a352e4 into bcd9e3e7285b4b1f8ce668f410f8b772(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T16:15:45,213 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 63f43c63344ece03d182c42497a352e4: 2024-11-23T16:15:45,213 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., storeName=63f43c63344ece03d182c42497a352e4/A, priority=13, startTime=1732378544699; duration=0sec 2024-11-23T16:15:45,213 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T16:15:45,213 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 63f43c63344ece03d182c42497a352e4:A 2024-11-23T16:15:45,228 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:45,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39788 deadline: 1732378605218, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:45,229 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:45,229 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:45,229 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:45,229 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:15:45,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39740 deadline: 1732378605218, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:45,229 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:45,229 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:45,229 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:45,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39720 deadline: 1732378605218, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:45,229 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:15:45,229 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:45,229 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:45,229 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:15:45,235 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:45,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39764 deadline: 1732378605228, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:45,236 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:45,236 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:45,236 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:45,236 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:15:45,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39752 deadline: 1732378605228, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:45,236 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:45,236 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:45,236 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:15:45,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-23T16:15:45,409 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-23T16:15:45,409 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-23T16:15:45,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] procedure2.ProcedureExecutor(1139): Stored pid=115, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees 2024-11-23T16:15:45,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-23T16:15:45,411 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=115, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-23T16:15:45,412 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=115, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-23T16:15:45,412 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=116, ppid=115, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-23T16:15:45,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-23T16:15:45,543 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:45,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39788 deadline: 1732378605537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:45,544 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:45,544 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:45,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39740 deadline: 1732378605538, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:45,544 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:45,544 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:15:45,544 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:45,544 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:45,544 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:45,544 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:15:45,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39720 deadline: 1732378605538, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:45,544 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:45,544 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:45,544 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:15:45,560 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:45,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39764 deadline: 1732378605548, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:45,560 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:45,560 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:45,560 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:45,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39752 deadline: 1732378605548, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:45,560 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:15:45,560 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:45,560 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:45,560 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:15:45,564 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-23T16:15:45,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4. 2024-11-23T16:15:45,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4. as already flushing 2024-11-23T16:15:45,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4. 2024-11-23T16:15:45,564 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 63f43c63344ece03d182c42497a352e4, NAME => 'TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:15:45,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 63f43c63344ece03d182c42497a352e4, NAME => 'TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:15:45,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 63f43c63344ece03d182c42497a352e4, NAME => 'TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 63f43c63344ece03d182c42497a352e4, NAME => 'TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:15:45,598 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=89 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/.tmp/B/e6778eae86ab40e7bbb9e5e24c3abf08 2024-11-23T16:15:45,605 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/.tmp/C/9f3f57eff44c48adbd1caffd0ce3b246 is 50, key is test_row_0/C:col10/1732378544733/Put/seqid=0 2024-11-23T16:15:45,619 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742263_1439 (size=12001) 2024-11-23T16:15:45,717 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-23T16:15:45,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4. 2024-11-23T16:15:45,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4. as already flushing 2024-11-23T16:15:45,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4. 2024-11-23T16:15:45,717 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 63f43c63344ece03d182c42497a352e4, NAME => 'TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:15:45,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 63f43c63344ece03d182c42497a352e4, NAME => 'TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:15:45,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 63f43c63344ece03d182c42497a352e4, NAME => 'TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 63f43c63344ece03d182c42497a352e4, NAME => 'TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:15:45,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-23T16:15:45,870 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-23T16:15:45,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4. 2024-11-23T16:15:45,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4. as already flushing 2024-11-23T16:15:45,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4. 2024-11-23T16:15:45,870 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 63f43c63344ece03d182c42497a352e4, NAME => 'TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:15:45,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 63f43c63344ece03d182c42497a352e4, NAME => 'TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:15:45,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 63f43c63344ece03d182c42497a352e4, NAME => 'TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 63f43c63344ece03d182c42497a352e4, NAME => 'TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:15:46,019 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=89 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/.tmp/C/9f3f57eff44c48adbd1caffd0ce3b246 2024-11-23T16:15:46,024 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/.tmp/A/83acfcade2b6405e88a009b9937c1f29 as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/A/83acfcade2b6405e88a009b9937c1f29 2024-11-23T16:15:46,024 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-23T16:15:46,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4. 2024-11-23T16:15:46,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4. as already flushing 2024-11-23T16:15:46,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4. 2024-11-23T16:15:46,025 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 63f43c63344ece03d182c42497a352e4, NAME => 'TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:15:46,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 63f43c63344ece03d182c42497a352e4, NAME => 'TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:15:46,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 63f43c63344ece03d182c42497a352e4, NAME => 'TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 63f43c63344ece03d182c42497a352e4, NAME => 'TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:15:46,028 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/A/83acfcade2b6405e88a009b9937c1f29, entries=200, sequenceid=89, filesize=14.0 K 2024-11-23T16:15:46,029 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/.tmp/B/e6778eae86ab40e7bbb9e5e24c3abf08 as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/B/e6778eae86ab40e7bbb9e5e24c3abf08 2024-11-23T16:15:46,032 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/B/e6778eae86ab40e7bbb9e5e24c3abf08, entries=150, sequenceid=89, filesize=11.7 K 2024-11-23T16:15:46,033 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/.tmp/C/9f3f57eff44c48adbd1caffd0ce3b246 as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/C/9f3f57eff44c48adbd1caffd0ce3b246 2024-11-23T16:15:46,037 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/C/9f3f57eff44c48adbd1caffd0ce3b246, entries=150, sequenceid=89, filesize=11.7 K 2024-11-23T16:15:46,037 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=140.89 KB/144270 for 63f43c63344ece03d182c42497a352e4 in 1272ms, sequenceid=89, compaction requested=false 2024-11-23T16:15:46,037 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 63f43c63344ece03d182c42497a352e4: 2024-11-23T16:15:46,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-23T16:15:46,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(8855): Flush requested on 63f43c63344ece03d182c42497a352e4 2024-11-23T16:15:46,059 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 63f43c63344ece03d182c42497a352e4 3/3 column families, dataSize=154.31 KB heapSize=426 KB 2024-11-23T16:15:46,059 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 63f43c63344ece03d182c42497a352e4, store=A 2024-11-23T16:15:46,059 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-23T16:15:46,060 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 63f43c63344ece03d182c42497a352e4, store=B 2024-11-23T16:15:46,060 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-23T16:15:46,060 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 63f43c63344ece03d182c42497a352e4, store=C 2024-11-23T16:15:46,060 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-23T16:15:46,065 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/.tmp/A/aa62cb4d2fe7489dbe2e53148271e649 is 50, key is test_row_0/A:col10/1732378544887/Put/seqid=0 2024-11-23T16:15:46,079 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742264_1440 (size=16781) 2024-11-23T16:15:46,080 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:46,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39788 deadline: 1732378606067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:46,081 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:46,081 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:46,081 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:46,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39752 deadline: 1732378606068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:46,081 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:15:46,081 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:46,081 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:46,081 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:46,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39764 deadline: 1732378606068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:46,081 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:15:46,082 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:46,082 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:46,082 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:15:46,083 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:46,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39740 deadline: 1732378606074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:46,084 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:46,084 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:46,084 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:15:46,089 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:46,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39720 deadline: 1732378606081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:46,089 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:46,089 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:46,089 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:15:46,178 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-23T16:15:46,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4. 2024-11-23T16:15:46,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4. as already flushing 2024-11-23T16:15:46,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4. 2024-11-23T16:15:46,178 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 63f43c63344ece03d182c42497a352e4, NAME => 'TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:15:46,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 63f43c63344ece03d182c42497a352e4, NAME => 'TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:15:46,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 63f43c63344ece03d182c42497a352e4, NAME => 'TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 63f43c63344ece03d182c42497a352e4, NAME => 'TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:15:46,195 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:46,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39788 deadline: 1732378606188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:46,195 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:46,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39740 deadline: 1732378606188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:46,195 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:46,195 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:46,195 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:15:46,196 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:46,196 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:46,196 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:15:46,210 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:46,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39720 deadline: 1732378606198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:46,210 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:46,210 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:46,210 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:15:46,331 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-23T16:15:46,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4. 2024-11-23T16:15:46,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4. as already flushing 2024-11-23T16:15:46,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4. 2024-11-23T16:15:46,332 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 63f43c63344ece03d182c42497a352e4, NAME => 'TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:15:46,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 63f43c63344ece03d182c42497a352e4, NAME => 'TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:15:46,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 63f43c63344ece03d182c42497a352e4, NAME => 'TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 63f43c63344ece03d182c42497a352e4, NAME => 'TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:15:46,406 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:46,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39788 deadline: 1732378606398, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:46,407 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:46,407 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:46,407 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:15:46,418 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:46,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39740 deadline: 1732378606408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:46,418 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:46,418 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:46,418 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:15:46,426 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:46,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39720 deadline: 1732378606418, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:46,426 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:46,426 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:46,426 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:15:46,479 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=130 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/.tmp/A/aa62cb4d2fe7489dbe2e53148271e649 2024-11-23T16:15:46,484 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-23T16:15:46,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4. 2024-11-23T16:15:46,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4. as already flushing 2024-11-23T16:15:46,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4. 2024-11-23T16:15:46,484 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 63f43c63344ece03d182c42497a352e4, NAME => 'TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:15:46,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 63f43c63344ece03d182c42497a352e4, NAME => 'TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:15:46,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 63f43c63344ece03d182c42497a352e4, NAME => 'TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 63f43c63344ece03d182c42497a352e4, NAME => 'TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:15:46,487 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/.tmp/B/777861f36c1c48479784df932cec97ce is 50, key is test_row_0/B:col10/1732378544887/Put/seqid=0 2024-11-23T16:15:46,490 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742265_1441 (size=12101) 2024-11-23T16:15:46,490 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=130 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/.tmp/B/777861f36c1c48479784df932cec97ce 2024-11-23T16:15:46,497 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/.tmp/C/222fcf65d0e34f709f6bc09e0d98cb8c is 50, key is test_row_0/C:col10/1732378544887/Put/seqid=0 2024-11-23T16:15:46,515 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742266_1442 (size=12101) 2024-11-23T16:15:46,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-23T16:15:46,637 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-23T16:15:46,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4. 2024-11-23T16:15:46,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4. as already flushing 2024-11-23T16:15:46,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4. 2024-11-23T16:15:46,638 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 63f43c63344ece03d182c42497a352e4, NAME => 'TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:15:46,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 63f43c63344ece03d182c42497a352e4, NAME => 'TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:15:46,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 63f43c63344ece03d182c42497a352e4, NAME => 'TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 63f43c63344ece03d182c42497a352e4, NAME => 'TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:15:46,727 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:46,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39788 deadline: 1732378606718, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:46,728 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:46,728 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:46,728 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:15:46,731 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:46,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39740 deadline: 1732378606728, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:46,731 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:46,731 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:46,731 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:46,731 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:15:46,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39720 deadline: 1732378606728, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:46,732 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:46,732 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:46,732 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:15:46,790 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-23T16:15:46,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4. 2024-11-23T16:15:46,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4. as already flushing 2024-11-23T16:15:46,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4. 2024-11-23T16:15:46,791 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 63f43c63344ece03d182c42497a352e4, NAME => 'TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:15:46,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 63f43c63344ece03d182c42497a352e4, NAME => 'TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:15:46,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 63f43c63344ece03d182c42497a352e4, NAME => 'TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 63f43c63344ece03d182c42497a352e4, NAME => 'TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:15:46,916 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=130 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/.tmp/C/222fcf65d0e34f709f6bc09e0d98cb8c 2024-11-23T16:15:46,926 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/.tmp/A/aa62cb4d2fe7489dbe2e53148271e649 as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/A/aa62cb4d2fe7489dbe2e53148271e649 2024-11-23T16:15:46,930 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/A/aa62cb4d2fe7489dbe2e53148271e649, entries=250, sequenceid=130, filesize=16.4 K 2024-11-23T16:15:46,931 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/.tmp/B/777861f36c1c48479784df932cec97ce as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/B/777861f36c1c48479784df932cec97ce 2024-11-23T16:15:46,934 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/B/777861f36c1c48479784df932cec97ce, entries=150, sequenceid=130, filesize=11.8 K 2024-11-23T16:15:46,935 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/.tmp/C/222fcf65d0e34f709f6bc09e0d98cb8c as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/C/222fcf65d0e34f709f6bc09e0d98cb8c 2024-11-23T16:15:46,941 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/C/222fcf65d0e34f709f6bc09e0d98cb8c, entries=150, sequenceid=130, filesize=11.8 K 2024-11-23T16:15:46,942 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-23T16:15:46,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4. 2024-11-23T16:15:46,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4. as already flushing 2024-11-23T16:15:46,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4. 2024-11-23T16:15:46,943 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 63f43c63344ece03d182c42497a352e4, NAME => 'TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:15:46,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 63f43c63344ece03d182c42497a352e4, NAME => 'TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:15:46,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 63f43c63344ece03d182c42497a352e4, NAME => 'TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 63f43c63344ece03d182c42497a352e4, NAME => 'TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:15:46,945 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~425.95 KB/436176, currentSize=33.54 KB/34350 for 63f43c63344ece03d182c42497a352e4 in 885ms, sequenceid=130, compaction requested=true 2024-11-23T16:15:46,945 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 63f43c63344ece03d182c42497a352e4: 2024-11-23T16:15:46,945 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T16:15:46,945 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 63f43c63344ece03d182c42497a352e4:A, priority=-2147483648, current under compaction store size is 1 2024-11-23T16:15:46,945 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T16:15:46,946 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T16:15:46,946 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 43329 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T16:15:46,946 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HStore(1541): 63f43c63344ece03d182c42497a352e4/A is initiating minor compaction (all files) 2024-11-23T16:15:46,946 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 63f43c63344ece03d182c42497a352e4/A in TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4. 2024-11-23T16:15:46,946 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/A/bcd9e3e7285b4b1f8ce668f410f8b772, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/A/83acfcade2b6405e88a009b9937c1f29, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/A/aa62cb4d2fe7489dbe2e53148271e649] into tmpdir=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/.tmp, totalSize=42.3 K 2024-11-23T16:15:46,946 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 63f43c63344ece03d182c42497a352e4:B, priority=-2147483648, current under compaction store size is 2 2024-11-23T16:15:46,946 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.Compactor(225): Compacting bcd9e3e7285b4b1f8ce668f410f8b772, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1732378544667 2024-11-23T16:15:46,946 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T16:15:46,947 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.Compactor(225): Compacting 83acfcade2b6405e88a009b9937c1f29, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=89, earliestPutTs=1732378544717 2024-11-23T16:15:46,947 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36309 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T16:15:46,947 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HStore(1541): 63f43c63344ece03d182c42497a352e4/B is initiating minor compaction (all files) 2024-11-23T16:15:46,947 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 63f43c63344ece03d182c42497a352e4/B in TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4. 2024-11-23T16:15:46,947 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/B/79f4376771074b9ea9180bc24137a064, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/B/e6778eae86ab40e7bbb9e5e24c3abf08, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/B/777861f36c1c48479784df932cec97ce] into tmpdir=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/.tmp, totalSize=35.5 K 2024-11-23T16:15:46,947 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 63f43c63344ece03d182c42497a352e4:C, priority=-2147483648, current under compaction store size is 3 2024-11-23T16:15:46,947 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.Compactor(225): Compacting aa62cb4d2fe7489dbe2e53148271e649, keycount=250, bloomtype=ROW, size=16.4 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1732378544876 2024-11-23T16:15:46,947 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.Compactor(225): Compacting 79f4376771074b9ea9180bc24137a064, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1732378544667 2024-11-23T16:15:46,948 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T16:15:46,948 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.Compactor(225): Compacting e6778eae86ab40e7bbb9e5e24c3abf08, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=89, earliestPutTs=1732378544733 2024-11-23T16:15:46,948 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.Compactor(225): Compacting 777861f36c1c48479784df932cec97ce, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1732378544887 2024-11-23T16:15:46,959 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 63f43c63344ece03d182c42497a352e4#A#compaction#368 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T16:15:46,963 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/.tmp/A/d64612b7e9af45218c8c57a8af10c624 is 50, key is test_row_0/A:col10/1732378544887/Put/seqid=0 2024-11-23T16:15:46,969 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 63f43c63344ece03d182c42497a352e4#B#compaction#369 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T16:15:46,970 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/.tmp/B/d61c7db00f1f4f33b1eb6a71074e89dc is 50, key is test_row_0/B:col10/1732378544887/Put/seqid=0 2024-11-23T16:15:46,982 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742267_1443 (size=12409) 2024-11-23T16:15:46,982 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742268_1444 (size=12409) 2024-11-23T16:15:46,989 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/.tmp/A/d64612b7e9af45218c8c57a8af10c624 as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/A/d64612b7e9af45218c8c57a8af10c624 2024-11-23T16:15:46,996 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 63f43c63344ece03d182c42497a352e4/A of 63f43c63344ece03d182c42497a352e4 into d64612b7e9af45218c8c57a8af10c624(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T16:15:46,996 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 63f43c63344ece03d182c42497a352e4: 2024-11-23T16:15:46,996 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., storeName=63f43c63344ece03d182c42497a352e4/A, priority=13, startTime=1732378546945; duration=0sec 2024-11-23T16:15:46,996 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T16:15:46,996 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 63f43c63344ece03d182c42497a352e4:A 2024-11-23T16:15:46,996 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T16:15:46,997 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36275 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T16:15:46,997 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HStore(1541): 63f43c63344ece03d182c42497a352e4/C is initiating minor compaction (all files) 2024-11-23T16:15:46,997 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 63f43c63344ece03d182c42497a352e4/C in TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4. 2024-11-23T16:15:46,997 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/C/457649d6dd8b4269824f8f975d3c4c42, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/C/9f3f57eff44c48adbd1caffd0ce3b246, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/C/222fcf65d0e34f709f6bc09e0d98cb8c] into tmpdir=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/.tmp, totalSize=35.4 K 2024-11-23T16:15:46,998 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.Compactor(225): Compacting 457649d6dd8b4269824f8f975d3c4c42, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1732378544667 2024-11-23T16:15:46,998 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9f3f57eff44c48adbd1caffd0ce3b246, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=89, earliestPutTs=1732378544733 2024-11-23T16:15:46,998 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.Compactor(225): Compacting 222fcf65d0e34f709f6bc09e0d98cb8c, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1732378544887 2024-11-23T16:15:47,006 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 63f43c63344ece03d182c42497a352e4#C#compaction#370 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T16:15:47,007 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/.tmp/C/77ae4cd18d95457a800dce425811621a is 50, key is test_row_0/C:col10/1732378544887/Put/seqid=0 2024-11-23T16:15:47,022 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742269_1445 (size=12375) 2024-11-23T16:15:47,096 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-23T16:15:47,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4. 2024-11-23T16:15:47,096 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2902): Flushing 63f43c63344ece03d182c42497a352e4 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-11-23T16:15:47,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 63f43c63344ece03d182c42497a352e4, store=A 2024-11-23T16:15:47,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-23T16:15:47,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 63f43c63344ece03d182c42497a352e4, store=B 2024-11-23T16:15:47,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-23T16:15:47,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 63f43c63344ece03d182c42497a352e4, store=C 2024-11-23T16:15:47,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-23T16:15:47,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(8855): Flush requested on 63f43c63344ece03d182c42497a352e4 2024-11-23T16:15:47,097 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4. as already flushing 2024-11-23T16:15:47,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/.tmp/A/230b2f3c02c84defb484385b5a644f0e is 50, key is test_row_0/A:col10/1732378547090/Put/seqid=0 2024-11-23T16:15:47,104 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742270_1446 (size=12151) 2024-11-23T16:15:47,180 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1a7c5470b472:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732378547122, max timestamp=1732378547176Num uniques 859; 2024-11-23T16:15:47,180 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1a7c5470b472:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732378547122, max timestamp=1732378547176Num uniques 859; 2024-11-23T16:15:47,180 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1a7c5470b472:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-23T16:15:47,180 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1a7c5470b472:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732378547122, max timestamp=1732378547176Num uniques 859; 2024-11-23T16:15:47,180 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1a7c5470b472:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-23T16:15:47,180 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1a7c5470b472:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-23T16:15:47,262 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:47,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39788 deadline: 1732378607242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:47,263 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:47,263 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:47,263 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:47,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39740 deadline: 1732378607243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:47,263 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:15:47,263 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:47,263 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:47,263 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:15:47,263 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:47,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39720 deadline: 1732378607245, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:47,263 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:47,263 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:47,263 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:15:47,263 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:47,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39752 deadline: 1732378607255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:47,264 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:47,264 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:47,264 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:15:47,269 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:47,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39764 deadline: 1732378607256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:47,270 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:47,270 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:47,270 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:15:47,380 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:47,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39788 deadline: 1732378607368, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:47,381 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:47,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39740 deadline: 1732378607368, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:47,381 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:47,381 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:47,381 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:15:47,381 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:47,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39720 deadline: 1732378607368, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:47,381 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:47,381 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:47,381 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:15:47,381 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:47,381 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:47,381 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:47,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39752 deadline: 1732378607368, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:47,381 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:15:47,381 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:47,381 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:47,381 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:15:47,385 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:47,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39764 deadline: 1732378607378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:47,385 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:47,385 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:47,385 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:15:47,387 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/.tmp/B/d61c7db00f1f4f33b1eb6a71074e89dc as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/B/d61c7db00f1f4f33b1eb6a71074e89dc 2024-11-23T16:15:47,390 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 63f43c63344ece03d182c42497a352e4/B of 63f43c63344ece03d182c42497a352e4 into d61c7db00f1f4f33b1eb6a71074e89dc(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T16:15:47,390 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 63f43c63344ece03d182c42497a352e4: 2024-11-23T16:15:47,390 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., storeName=63f43c63344ece03d182c42497a352e4/B, priority=13, startTime=1732378546945; duration=0sec 2024-11-23T16:15:47,390 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T16:15:47,390 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 63f43c63344ece03d182c42497a352e4:B 2024-11-23T16:15:47,426 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/.tmp/C/77ae4cd18d95457a800dce425811621a as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/C/77ae4cd18d95457a800dce425811621a 2024-11-23T16:15:47,430 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 63f43c63344ece03d182c42497a352e4/C of 63f43c63344ece03d182c42497a352e4 into 77ae4cd18d95457a800dce425811621a(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T16:15:47,430 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 63f43c63344ece03d182c42497a352e4: 2024-11-23T16:15:47,430 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., storeName=63f43c63344ece03d182c42497a352e4/C, priority=13, startTime=1732378546946; duration=0sec 2024-11-23T16:15:47,430 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T16:15:47,430 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 63f43c63344ece03d182c42497a352e4:C 2024-11-23T16:15:47,504 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=140 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/.tmp/A/230b2f3c02c84defb484385b5a644f0e 2024-11-23T16:15:47,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/.tmp/B/55ed716d7c3e4d0b9645c4df1d8b566f is 50, key is test_row_0/B:col10/1732378547090/Put/seqid=0 2024-11-23T16:15:47,515 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742271_1447 (size=12151) 2024-11-23T16:15:47,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-23T16:15:47,601 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:47,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39788 deadline: 1732378607588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:47,601 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:47,601 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:47,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39740 deadline: 1732378607588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:47,601 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:47,601 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:15:47,602 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:47,602 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:47,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39720 deadline: 1732378607588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:47,602 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:47,602 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:15:47,602 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:47,602 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:47,602 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:47,602 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:15:47,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39752 deadline: 1732378607588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:47,602 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:47,602 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:47,602 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:47,602 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:15:47,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39764 deadline: 1732378607588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:47,603 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:47,603 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:47,603 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:15:47,916 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=140 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/.tmp/B/55ed716d7c3e4d0b9645c4df1d8b566f 2024-11-23T16:15:47,917 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:47,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39788 deadline: 1732378607908, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:47,917 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:47,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39740 deadline: 1732378607908, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:47,918 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:47,918 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:47,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39720 deadline: 1732378607908, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:47,918 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:47,918 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:47,918 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:15:47,918 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:47,918 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:15:47,918 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:47,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39752 deadline: 1732378607908, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:47,919 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:47,919 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:47,919 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:47,919 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:47,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39764 deadline: 1732378607908, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:47,919 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:47,919 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:15:47,919 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:15:47,919 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:47,919 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:47,919 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:15:47,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/.tmp/C/4b1dbabea1634cd7b198ae442084faa2 is 50, key is test_row_0/C:col10/1732378547090/Put/seqid=0 2024-11-23T16:15:47,927 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742272_1448 (size=12151) 2024-11-23T16:15:48,328 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=140 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/.tmp/C/4b1dbabea1634cd7b198ae442084faa2 2024-11-23T16:15:48,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/.tmp/A/230b2f3c02c84defb484385b5a644f0e as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/A/230b2f3c02c84defb484385b5a644f0e 2024-11-23T16:15:48,335 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/A/230b2f3c02c84defb484385b5a644f0e, entries=150, sequenceid=140, filesize=11.9 K 2024-11-23T16:15:48,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/.tmp/B/55ed716d7c3e4d0b9645c4df1d8b566f as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/B/55ed716d7c3e4d0b9645c4df1d8b566f 2024-11-23T16:15:48,339 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/B/55ed716d7c3e4d0b9645c4df1d8b566f, entries=150, sequenceid=140, filesize=11.9 K 2024-11-23T16:15:48,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/.tmp/C/4b1dbabea1634cd7b198ae442084faa2 as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/C/4b1dbabea1634cd7b198ae442084faa2 2024-11-23T16:15:48,343 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/C/4b1dbabea1634cd7b198ae442084faa2, entries=150, sequenceid=140, filesize=11.9 K 2024-11-23T16:15:48,344 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=140.89 KB/144270 for 63f43c63344ece03d182c42497a352e4 in 1247ms, sequenceid=140, compaction requested=false 2024-11-23T16:15:48,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2603): Flush status journal for 63f43c63344ece03d182c42497a352e4: 2024-11-23T16:15:48,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4. 2024-11-23T16:15:48,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=116 2024-11-23T16:15:48,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.HMaster(4169): Remote procedure done, pid=116 2024-11-23T16:15:48,346 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=116, resume processing ppid=115 2024-11-23T16:15:48,347 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=116, ppid=115, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.9330 sec 2024-11-23T16:15:48,348 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=115, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees in 2.9380 sec 2024-11-23T16:15:48,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(8855): Flush requested on 63f43c63344ece03d182c42497a352e4 2024-11-23T16:15:48,434 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 63f43c63344ece03d182c42497a352e4 3/3 column families, dataSize=147.60 KB heapSize=420.84 KB 2024-11-23T16:15:48,435 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 63f43c63344ece03d182c42497a352e4, store=A 2024-11-23T16:15:48,435 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-23T16:15:48,435 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 63f43c63344ece03d182c42497a352e4, store=B 2024-11-23T16:15:48,435 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-23T16:15:48,435 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 63f43c63344ece03d182c42497a352e4, store=C 2024-11-23T16:15:48,435 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-23T16:15:48,441 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/.tmp/A/3c5ff086e7164122a4757921e19087e8 is 50, key is test_row_0/A:col10/1732378548433/Put/seqid=0 2024-11-23T16:15:48,445 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742273_1449 (size=14541) 2024-11-23T16:15:48,453 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:48,453 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:48,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39740 deadline: 1732378608437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:48,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39752 deadline: 1732378608439, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:48,453 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:48,453 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:48,453 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:48,453 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:15:48,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39720 deadline: 1732378608439, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:48,453 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:48,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39764 deadline: 1732378608441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:48,454 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:48,454 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:48,454 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:48,454 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:15:48,454 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:48,454 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:15:48,454 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:48,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39788 deadline: 1732378608453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:48,454 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:48,454 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:48,454 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:15:48,454 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:48,454 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:48,454 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:15:48,561 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:48,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39740 deadline: 1732378608558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:48,561 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:48,561 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:48,561 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:48,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39752 deadline: 1732378608558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:48,561 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:15:48,562 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:48,562 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:48,562 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:48,562 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:15:48,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39720 deadline: 1732378608558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:48,562 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:48,562 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:48,562 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:48,562 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:15:48,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39788 deadline: 1732378608559, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:48,562 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:48,562 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:48,562 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:15:48,562 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:48,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39764 deadline: 1732378608559, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:48,563 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:48,563 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:48,563 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:15:48,771 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:48,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39740 deadline: 1732378608768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:48,771 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:48,772 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:48,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39752 deadline: 1732378608768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:48,772 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:48,772 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:15:48,772 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:48,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39720 deadline: 1732378608768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:48,772 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:48,772 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:48,772 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:15:48,772 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:48,772 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:48,772 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:48,772 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:15:48,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39788 deadline: 1732378608768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:48,772 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:48,772 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:48,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39764 deadline: 1732378608769, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:48,772 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:48,772 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:15:48,773 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:48,773 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:48,773 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:15:48,846 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=180 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/.tmp/A/3c5ff086e7164122a4757921e19087e8 2024-11-23T16:15:48,853 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/.tmp/B/5a5972fa91e048f9b9e307e54020cb97 is 50, key is test_row_0/B:col10/1732378548433/Put/seqid=0 2024-11-23T16:15:48,856 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742274_1450 (size=12151) 2024-11-23T16:15:49,088 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:49,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39740 deadline: 1732378609078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:49,088 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:49,088 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:49,088 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:15:49,088 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:49,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39752 deadline: 1732378609078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:49,089 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:49,089 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:49,089 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:49,089 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:15:49,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39720 deadline: 1732378609078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:49,089 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:49,089 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:49,089 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:49,089 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:15:49,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39788 deadline: 1732378609078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:49,089 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:49,089 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:49,089 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:15:49,089 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:49,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39764 deadline: 1732378609078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:49,090 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:49,090 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:49,090 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:15:49,256 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=180 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/.tmp/B/5a5972fa91e048f9b9e307e54020cb97 2024-11-23T16:15:49,263 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/.tmp/C/5a0ed7d8fd344976b7cc8f250d603503 is 50, key is test_row_0/C:col10/1732378548433/Put/seqid=0 2024-11-23T16:15:49,266 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742275_1451 (size=12151) 2024-11-23T16:15:49,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-23T16:15:49,578 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-23T16:15:49,579 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-23T16:15:49,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] procedure2.ProcedureExecutor(1139): Stored pid=117, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees 2024-11-23T16:15:49,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-23T16:15:49,580 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=117, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-23T16:15:49,581 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=117, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-23T16:15:49,581 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=118, ppid=117, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-23T16:15:49,609 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:49,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39740 deadline: 1732378609598, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:49,610 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:49,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39752 deadline: 1732378609598, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:49,610 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:49,610 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:49,610 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:15:49,610 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:49,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39720 deadline: 1732378609599, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:49,610 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:49,610 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:49,610 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:15:49,610 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:49,610 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:49,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39788 deadline: 1732378609599, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:49,610 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:49,610 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:15:49,611 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:49,611 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:49,611 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:49,611 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:15:49,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39764 deadline: 1732378609599, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:49,611 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:49,611 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:49,611 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:15:49,667 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=180 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/.tmp/C/5a0ed7d8fd344976b7cc8f250d603503 2024-11-23T16:15:49,672 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/.tmp/A/3c5ff086e7164122a4757921e19087e8 as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/A/3c5ff086e7164122a4757921e19087e8 2024-11-23T16:15:49,675 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/A/3c5ff086e7164122a4757921e19087e8, entries=200, sequenceid=180, filesize=14.2 K 2024-11-23T16:15:49,675 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/.tmp/B/5a5972fa91e048f9b9e307e54020cb97 as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/B/5a5972fa91e048f9b9e307e54020cb97 2024-11-23T16:15:49,678 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/B/5a5972fa91e048f9b9e307e54020cb97, entries=150, sequenceid=180, filesize=11.9 K 2024-11-23T16:15:49,678 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/.tmp/C/5a0ed7d8fd344976b7cc8f250d603503 as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/C/5a0ed7d8fd344976b7cc8f250d603503 2024-11-23T16:15:49,681 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/C/5a0ed7d8fd344976b7cc8f250d603503, entries=150, sequenceid=180, filesize=11.9 K 2024-11-23T16:15:49,682 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~420.80 KB/430896, currentSize=33.54 KB/34350 for 63f43c63344ece03d182c42497a352e4 in 1248ms, sequenceid=180, compaction requested=true 2024-11-23T16:15:49,682 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 63f43c63344ece03d182c42497a352e4: 2024-11-23T16:15:49,682 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 63f43c63344ece03d182c42497a352e4:A, priority=-2147483648, current under compaction store size is 1 2024-11-23T16:15:49,682 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T16:15:49,682 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T16:15:49,682 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T16:15:49,682 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 63f43c63344ece03d182c42497a352e4:B, priority=-2147483648, current under compaction store size is 2 2024-11-23T16:15:49,683 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T16:15:49,683 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 63f43c63344ece03d182c42497a352e4:C, priority=-2147483648, current under compaction store size is 3 2024-11-23T16:15:49,683 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T16:15:49,683 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36711 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T16:15:49,683 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39101 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T16:15:49,683 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HStore(1541): 63f43c63344ece03d182c42497a352e4/A is initiating minor compaction (all files) 2024-11-23T16:15:49,683 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HStore(1541): 63f43c63344ece03d182c42497a352e4/B is initiating minor compaction (all files) 2024-11-23T16:15:49,683 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 63f43c63344ece03d182c42497a352e4/A in TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4. 2024-11-23T16:15:49,683 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 63f43c63344ece03d182c42497a352e4/B in TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4. 2024-11-23T16:15:49,684 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/A/d64612b7e9af45218c8c57a8af10c624, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/A/230b2f3c02c84defb484385b5a644f0e, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/A/3c5ff086e7164122a4757921e19087e8] into tmpdir=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/.tmp, totalSize=38.2 K 2024-11-23T16:15:49,684 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/B/d61c7db00f1f4f33b1eb6a71074e89dc, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/B/55ed716d7c3e4d0b9645c4df1d8b566f, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/B/5a5972fa91e048f9b9e307e54020cb97] into tmpdir=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/.tmp, totalSize=35.9 K 2024-11-23T16:15:49,684 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.Compactor(225): Compacting d64612b7e9af45218c8c57a8af10c624, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1732378544887 2024-11-23T16:15:49,684 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.Compactor(225): Compacting d61c7db00f1f4f33b1eb6a71074e89dc, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1732378544887 2024-11-23T16:15:49,684 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.Compactor(225): Compacting 230b2f3c02c84defb484385b5a644f0e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=140, earliestPutTs=1732378546064 2024-11-23T16:15:49,684 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.Compactor(225): Compacting 55ed716d7c3e4d0b9645c4df1d8b566f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=140, earliestPutTs=1732378546064 2024-11-23T16:15:49,684 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.Compactor(225): Compacting 5a5972fa91e048f9b9e307e54020cb97, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=180, earliestPutTs=1732378547241 2024-11-23T16:15:49,684 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3c5ff086e7164122a4757921e19087e8, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=180, earliestPutTs=1732378547241 2024-11-23T16:15:49,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-23T16:15:49,692 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 63f43c63344ece03d182c42497a352e4#A#compaction#377 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-23T16:15:49,692 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/.tmp/A/72eae2668f414abd857521062e3d9477 is 50, key is test_row_0/A:col10/1732378548433/Put/seqid=0 2024-11-23T16:15:49,693 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 63f43c63344ece03d182c42497a352e4#B#compaction#378 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T16:15:49,693 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/.tmp/B/984a6841474d4eeeaf864a5552cfa3d2 is 50, key is test_row_0/B:col10/1732378548433/Put/seqid=0 2024-11-23T16:15:49,709 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742276_1452 (size=12561) 2024-11-23T16:15:49,710 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742277_1453 (size=12561) 2024-11-23T16:15:49,713 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/.tmp/A/72eae2668f414abd857521062e3d9477 as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/A/72eae2668f414abd857521062e3d9477 2024-11-23T16:15:49,717 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 63f43c63344ece03d182c42497a352e4/A of 63f43c63344ece03d182c42497a352e4 into 72eae2668f414abd857521062e3d9477(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T16:15:49,717 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 63f43c63344ece03d182c42497a352e4: 2024-11-23T16:15:49,717 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., storeName=63f43c63344ece03d182c42497a352e4/A, priority=13, startTime=1732378549682; duration=0sec 2024-11-23T16:15:49,717 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T16:15:49,717 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 63f43c63344ece03d182c42497a352e4:A 2024-11-23T16:15:49,717 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T16:15:49,718 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36677 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T16:15:49,718 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HStore(1541): 63f43c63344ece03d182c42497a352e4/C is initiating minor compaction (all files) 2024-11-23T16:15:49,718 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 63f43c63344ece03d182c42497a352e4/C in TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4. 2024-11-23T16:15:49,718 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/C/77ae4cd18d95457a800dce425811621a, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/C/4b1dbabea1634cd7b198ae442084faa2, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/C/5a0ed7d8fd344976b7cc8f250d603503] into tmpdir=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/.tmp, totalSize=35.8 K 2024-11-23T16:15:49,718 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.Compactor(225): Compacting 77ae4cd18d95457a800dce425811621a, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1732378544887 2024-11-23T16:15:49,719 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4b1dbabea1634cd7b198ae442084faa2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=140, earliestPutTs=1732378546064 2024-11-23T16:15:49,719 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5a0ed7d8fd344976b7cc8f250d603503, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=180, earliestPutTs=1732378547241 2024-11-23T16:15:49,726 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 63f43c63344ece03d182c42497a352e4#C#compaction#379 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T16:15:49,726 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/.tmp/C/3d648d05e92148d3a0e663b21f08e739 is 50, key is test_row_0/C:col10/1732378548433/Put/seqid=0 2024-11-23T16:15:49,733 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-23T16:15:49,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4. 2024-11-23T16:15:49,734 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2902): Flushing 63f43c63344ece03d182c42497a352e4 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-23T16:15:49,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 63f43c63344ece03d182c42497a352e4, store=A 2024-11-23T16:15:49,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-23T16:15:49,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 63f43c63344ece03d182c42497a352e4, store=B 2024-11-23T16:15:49,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-23T16:15:49,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 63f43c63344ece03d182c42497a352e4, store=C 2024-11-23T16:15:49,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-23T16:15:49,741 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742278_1454 (size=12527) 2024-11-23T16:15:49,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/.tmp/A/5e3be1e387b94f7c91c829c1071cadc1 is 50, key is test_row_1/A:col10/1732378548452/Put/seqid=0 2024-11-23T16:15:49,755 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/.tmp/C/3d648d05e92148d3a0e663b21f08e739 as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/C/3d648d05e92148d3a0e663b21f08e739 2024-11-23T16:15:49,761 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 63f43c63344ece03d182c42497a352e4/C of 63f43c63344ece03d182c42497a352e4 into 3d648d05e92148d3a0e663b21f08e739(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T16:15:49,761 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 63f43c63344ece03d182c42497a352e4: 2024-11-23T16:15:49,761 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., storeName=63f43c63344ece03d182c42497a352e4/C, priority=13, startTime=1732378549683; duration=0sec 2024-11-23T16:15:49,761 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T16:15:49,761 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 63f43c63344ece03d182c42497a352e4:C 2024-11-23T16:15:49,766 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742279_1455 (size=9757) 2024-11-23T16:15:49,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-23T16:15:50,108 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-23T16:15:50,108 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-23T16:15:50,115 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/.tmp/B/984a6841474d4eeeaf864a5552cfa3d2 as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/B/984a6841474d4eeeaf864a5552cfa3d2 2024-11-23T16:15:50,119 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 63f43c63344ece03d182c42497a352e4/B of 63f43c63344ece03d182c42497a352e4 into 984a6841474d4eeeaf864a5552cfa3d2(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T16:15:50,119 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 63f43c63344ece03d182c42497a352e4: 2024-11-23T16:15:50,119 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., storeName=63f43c63344ece03d182c42497a352e4/B, priority=13, startTime=1732378549682; duration=0sec 2024-11-23T16:15:50,119 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T16:15:50,119 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 63f43c63344ece03d182c42497a352e4:B 2024-11-23T16:15:50,167 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=189 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/.tmp/A/5e3be1e387b94f7c91c829c1071cadc1 2024-11-23T16:15:50,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/.tmp/B/779aa8a1e548449ba38570f80380f917 is 50, key is test_row_1/B:col10/1732378548452/Put/seqid=0 2024-11-23T16:15:50,185 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742280_1456 (size=9757) 2024-11-23T16:15:50,188 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=189 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/.tmp/B/779aa8a1e548449ba38570f80380f917 2024-11-23T16:15:50,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/.tmp/C/cabd0470aaaa494b8278d295b608ab28 is 50, key is test_row_1/C:col10/1732378548452/Put/seqid=0 2024-11-23T16:15:50,198 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742281_1457 (size=9757) 2024-11-23T16:15:50,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-23T16:15:50,599 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=189 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/.tmp/C/cabd0470aaaa494b8278d295b608ab28 2024-11-23T16:15:50,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/.tmp/A/5e3be1e387b94f7c91c829c1071cadc1 as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/A/5e3be1e387b94f7c91c829c1071cadc1 2024-11-23T16:15:50,607 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/A/5e3be1e387b94f7c91c829c1071cadc1, entries=100, sequenceid=189, filesize=9.5 K 2024-11-23T16:15:50,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/.tmp/B/779aa8a1e548449ba38570f80380f917 as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/B/779aa8a1e548449ba38570f80380f917 2024-11-23T16:15:50,612 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/B/779aa8a1e548449ba38570f80380f917, entries=100, sequenceid=189, filesize=9.5 K 2024-11-23T16:15:50,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/.tmp/C/cabd0470aaaa494b8278d295b608ab28 as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/C/cabd0470aaaa494b8278d295b608ab28 2024-11-23T16:15:50,616 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/C/cabd0470aaaa494b8278d295b608ab28, entries=100, sequenceid=189, filesize=9.5 K 2024-11-23T16:15:50,617 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 63f43c63344ece03d182c42497a352e4 in 884ms, sequenceid=189, compaction requested=false 2024-11-23T16:15:50,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2603): Flush status journal for 63f43c63344ece03d182c42497a352e4: 2024-11-23T16:15:50,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4. 2024-11-23T16:15:50,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=118 2024-11-23T16:15:50,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.HMaster(4169): Remote procedure done, pid=118 2024-11-23T16:15:50,624 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=118, resume processing ppid=117 2024-11-23T16:15:50,624 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=118, ppid=117, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0380 sec 2024-11-23T16:15:50,626 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=117, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees in 1.0460 sec 2024-11-23T16:15:50,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(8855): Flush requested on 63f43c63344ece03d182c42497a352e4 2024-11-23T16:15:50,628 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 63f43c63344ece03d182c42497a352e4 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-23T16:15:50,629 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 63f43c63344ece03d182c42497a352e4, store=A 2024-11-23T16:15:50,629 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-23T16:15:50,629 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 63f43c63344ece03d182c42497a352e4, store=B 2024-11-23T16:15:50,629 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-23T16:15:50,629 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 63f43c63344ece03d182c42497a352e4, store=C 2024-11-23T16:15:50,629 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-23T16:15:50,632 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/.tmp/A/b712b5711ed4432587b0c6c286d24466 is 50, key is test_row_0/A:col10/1732378550626/Put/seqid=0 2024-11-23T16:15:50,636 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742282_1458 (size=14541) 2024-11-23T16:15:50,636 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=202 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/.tmp/A/b712b5711ed4432587b0c6c286d24466 2024-11-23T16:15:50,643 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/.tmp/B/cc5c598fd8124740ae4b3a70355f0561 is 50, key is test_row_0/B:col10/1732378550626/Put/seqid=0 2024-11-23T16:15:50,646 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742283_1459 (size=12151) 2024-11-23T16:15:50,669 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:50,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39740 deadline: 1732378610655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:50,669 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:50,669 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:50,669 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:50,669 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:15:50,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39788 deadline: 1732378610656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:50,670 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:50,670 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:50,670 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:15:50,671 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1a7c5470b472:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732378550648, max timestamp=1732378550668Num uniques 859; 2024-11-23T16:15:50,671 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1a7c5470b472:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-23T16:15:50,671 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1a7c5470b472:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732378550648, max timestamp=1732378550668Num uniques 859; 2024-11-23T16:15:50,671 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1a7c5470b472:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-23T16:15:50,674 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1a7c5470b472:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732378550648, max timestamp=1732378550668Num uniques 859; 2024-11-23T16:15:50,674 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1a7c5470b472:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-23T16:15:50,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-23T16:15:50,719 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-23T16:15:50,719 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-23T16:15:50,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] procedure2.ProcedureExecutor(1139): Stored pid=119, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees 2024-11-23T16:15:50,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-23T16:15:50,721 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=119, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-23T16:15:50,722 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=119, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-23T16:15:50,722 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=120, ppid=119, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-23T16:15:50,733 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:50,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39764 deadline: 1732378610719, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:50,733 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:50,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39752 deadline: 1732378610720, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:50,733 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:50,733 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:50,733 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:15:50,734 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:50,734 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:50,734 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:15:50,744 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:50,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39720 deadline: 1732378610731, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:50,744 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:50,744 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:50,744 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:15:50,779 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:50,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39740 deadline: 1732378610778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:50,780 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:50,780 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:50,780 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:15:50,790 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:50,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39788 deadline: 1732378610778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:50,790 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:50,790 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:50,790 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:15:50,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-23T16:15:50,840 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:50,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39764 deadline: 1732378610837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:50,841 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:50,841 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:50,841 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:50,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39752 deadline: 1732378610838, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:50,841 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:15:50,841 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:50,841 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:50,841 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:15:50,856 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:50,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39720 deadline: 1732378610848, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:50,857 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:50,857 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:50,857 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:15:50,874 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-23T16:15:50,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4. 2024-11-23T16:15:50,875 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4. as already flushing 2024-11-23T16:15:50,875 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4. 2024-11-23T16:15:50,875 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 63f43c63344ece03d182c42497a352e4, NAME => 'TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:15:50,875 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 63f43c63344ece03d182c42497a352e4, NAME => 'TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:15:50,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 63f43c63344ece03d182c42497a352e4, NAME => 'TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 63f43c63344ece03d182c42497a352e4, NAME => 'TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:15:50,998 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:50,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39740 deadline: 1732378610988, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:50,999 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:50,999 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:50,999 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:15:51,005 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:51,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39788 deadline: 1732378610997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:51,005 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:51,005 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:51,005 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:15:51,027 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-23T16:15:51,028 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4. 2024-11-23T16:15:51,028 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4. as already flushing 2024-11-23T16:15:51,028 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4. 2024-11-23T16:15:51,028 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 63f43c63344ece03d182c42497a352e4, NAME => 'TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:15:51,028 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 63f43c63344ece03d182c42497a352e4, NAME => 'TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:15:51,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 63f43c63344ece03d182c42497a352e4, NAME => 'TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 63f43c63344ece03d182c42497a352e4, NAME => 'TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:15:51,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-23T16:15:51,047 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=202 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/.tmp/B/cc5c598fd8124740ae4b3a70355f0561 2024-11-23T16:15:51,053 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/.tmp/C/7a7b449f55c84b91a8a38c77e259b635 is 50, key is test_row_0/C:col10/1732378550626/Put/seqid=0 2024-11-23T16:15:51,056 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742284_1460 (size=12151) 2024-11-23T16:15:51,057 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:51,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39764 deadline: 1732378611048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:51,059 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:51,059 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:51,059 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:15:51,059 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:51,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39752 deadline: 1732378611048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:51,059 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:51,059 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:51,059 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:15:51,072 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:51,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39720 deadline: 1732378611067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:51,072 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:51,072 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:51,072 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:15:51,180 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-23T16:15:51,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4. 2024-11-23T16:15:51,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4. as already flushing 2024-11-23T16:15:51,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4. 2024-11-23T16:15:51,181 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 63f43c63344ece03d182c42497a352e4, NAME => 'TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:15:51,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 63f43c63344ece03d182c42497a352e4, NAME => 'TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:15:51,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 63f43c63344ece03d182c42497a352e4, NAME => 'TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 63f43c63344ece03d182c42497a352e4, NAME => 'TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:15:51,315 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:51,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39740 deadline: 1732378611308, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:51,315 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:51,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39788 deadline: 1732378611308, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:51,315 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:51,316 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:51,316 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:15:51,316 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:51,316 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:51,316 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:15:51,333 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-23T16:15:51,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4. 2024-11-23T16:15:51,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4. as already flushing 2024-11-23T16:15:51,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4. 2024-11-23T16:15:51,334 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 63f43c63344ece03d182c42497a352e4, NAME => 'TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:15:51,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 63f43c63344ece03d182c42497a352e4, NAME => 'TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:15:51,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 63f43c63344ece03d182c42497a352e4, NAME => 'TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 63f43c63344ece03d182c42497a352e4, NAME => 'TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:15:51,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-23T16:15:51,377 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:51,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39764 deadline: 1732378611368, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:51,377 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:51,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39752 deadline: 1732378611368, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:51,378 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:51,378 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:51,378 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:15:51,378 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:51,378 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:51,378 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:15:51,389 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:51,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39720 deadline: 1732378611378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:51,389 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:51,389 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:51,389 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:15:51,457 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=202 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/.tmp/C/7a7b449f55c84b91a8a38c77e259b635 2024-11-23T16:15:51,460 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/.tmp/A/b712b5711ed4432587b0c6c286d24466 as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/A/b712b5711ed4432587b0c6c286d24466 2024-11-23T16:15:51,463 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/A/b712b5711ed4432587b0c6c286d24466, entries=200, sequenceid=202, filesize=14.2 K 2024-11-23T16:15:51,464 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/.tmp/B/cc5c598fd8124740ae4b3a70355f0561 as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/B/cc5c598fd8124740ae4b3a70355f0561 2024-11-23T16:15:51,467 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/B/cc5c598fd8124740ae4b3a70355f0561, entries=150, sequenceid=202, filesize=11.9 K 2024-11-23T16:15:51,468 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/.tmp/C/7a7b449f55c84b91a8a38c77e259b635 as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/C/7a7b449f55c84b91a8a38c77e259b635 2024-11-23T16:15:51,471 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/C/7a7b449f55c84b91a8a38c77e259b635, entries=150, sequenceid=202, filesize=11.9 K 2024-11-23T16:15:51,471 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 63f43c63344ece03d182c42497a352e4 in 843ms, sequenceid=202, compaction requested=true 2024-11-23T16:15:51,471 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 63f43c63344ece03d182c42497a352e4: 2024-11-23T16:15:51,472 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 63f43c63344ece03d182c42497a352e4:A, priority=-2147483648, current under compaction store size is 1 2024-11-23T16:15:51,472 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T16:15:51,472 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 63f43c63344ece03d182c42497a352e4:B, priority=-2147483648, current under compaction store size is 2 2024-11-23T16:15:51,472 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T16:15:51,472 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 63f43c63344ece03d182c42497a352e4:C, priority=-2147483648, current under compaction store size is 3 2024-11-23T16:15:51,472 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T16:15:51,472 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T16:15:51,472 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T16:15:51,472 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36859 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T16:15:51,472 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34469 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T16:15:51,472 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HStore(1541): 63f43c63344ece03d182c42497a352e4/B is initiating minor compaction (all files) 2024-11-23T16:15:51,472 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HStore(1541): 63f43c63344ece03d182c42497a352e4/A is initiating minor compaction (all files) 2024-11-23T16:15:51,472 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 63f43c63344ece03d182c42497a352e4/B in TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4. 2024-11-23T16:15:51,473 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 63f43c63344ece03d182c42497a352e4/A in TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4. 2024-11-23T16:15:51,473 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/B/984a6841474d4eeeaf864a5552cfa3d2, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/B/779aa8a1e548449ba38570f80380f917, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/B/cc5c598fd8124740ae4b3a70355f0561] into tmpdir=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/.tmp, totalSize=33.7 K 2024-11-23T16:15:51,473 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/A/72eae2668f414abd857521062e3d9477, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/A/5e3be1e387b94f7c91c829c1071cadc1, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/A/b712b5711ed4432587b0c6c286d24466] into tmpdir=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/.tmp, totalSize=36.0 K 2024-11-23T16:15:51,473 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.Compactor(225): Compacting 984a6841474d4eeeaf864a5552cfa3d2, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=180, earliestPutTs=1732378547241 2024-11-23T16:15:51,473 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.Compactor(225): Compacting 72eae2668f414abd857521062e3d9477, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=180, earliestPutTs=1732378547241 2024-11-23T16:15:51,473 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.Compactor(225): Compacting 779aa8a1e548449ba38570f80380f917, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=189, earliestPutTs=1732378548437 2024-11-23T16:15:51,473 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5e3be1e387b94f7c91c829c1071cadc1, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=189, earliestPutTs=1732378548437 2024-11-23T16:15:51,473 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.Compactor(225): Compacting cc5c598fd8124740ae4b3a70355f0561, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=202, earliestPutTs=1732378550626 2024-11-23T16:15:51,473 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.Compactor(225): Compacting b712b5711ed4432587b0c6c286d24466, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=202, earliestPutTs=1732378550624 2024-11-23T16:15:51,480 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 63f43c63344ece03d182c42497a352e4#A#compaction#386 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-23T16:15:51,481 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/.tmp/A/b75f3522fd4448da80b8cf120f65f080 is 50, key is test_row_0/A:col10/1732378550626/Put/seqid=0 2024-11-23T16:15:51,481 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 63f43c63344ece03d182c42497a352e4#B#compaction#387 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T16:15:51,482 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/.tmp/B/c469a19218cb41a5b69e98708c12c425 is 50, key is test_row_0/B:col10/1732378550626/Put/seqid=0 2024-11-23T16:15:51,485 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742285_1461 (size=12663) 2024-11-23T16:15:51,486 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-23T16:15:51,486 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4. 2024-11-23T16:15:51,486 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2902): Flushing 63f43c63344ece03d182c42497a352e4 3/3 column families, dataSize=127.47 KB heapSize=355.69 KB 2024-11-23T16:15:51,486 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 63f43c63344ece03d182c42497a352e4, store=A 2024-11-23T16:15:51,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-23T16:15:51,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 63f43c63344ece03d182c42497a352e4, store=B 2024-11-23T16:15:51,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-23T16:15:51,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 63f43c63344ece03d182c42497a352e4, store=C 2024-11-23T16:15:51,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-23T16:15:51,489 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742286_1462 (size=12663) 2024-11-23T16:15:51,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/.tmp/A/5e891158ee0a4643a9bfb4c8def3d58c is 50, key is test_row_0/A:col10/1732378550716/Put/seqid=0 2024-11-23T16:15:51,498 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742287_1463 (size=12151) 2024-11-23T16:15:51,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(8855): Flush requested on 63f43c63344ece03d182c42497a352e4 2024-11-23T16:15:51,835 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4. as already flushing 2024-11-23T16:15:51,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-23T16:15:51,864 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:51,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39740 deadline: 1732378611858, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:51,865 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:51,865 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:51,865 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:15:51,871 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:51,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39788 deadline: 1732378611861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:51,872 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:51,872 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:51,872 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:15:51,890 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/.tmp/A/b75f3522fd4448da80b8cf120f65f080 as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/A/b75f3522fd4448da80b8cf120f65f080 2024-11-23T16:15:51,893 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/.tmp/B/c469a19218cb41a5b69e98708c12c425 as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/B/c469a19218cb41a5b69e98708c12c425 2024-11-23T16:15:51,894 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 63f43c63344ece03d182c42497a352e4/A of 63f43c63344ece03d182c42497a352e4 into b75f3522fd4448da80b8cf120f65f080(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T16:15:51,894 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 63f43c63344ece03d182c42497a352e4: 2024-11-23T16:15:51,894 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., storeName=63f43c63344ece03d182c42497a352e4/A, priority=13, startTime=1732378551471; duration=0sec 2024-11-23T16:15:51,894 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T16:15:51,894 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 63f43c63344ece03d182c42497a352e4:A 2024-11-23T16:15:51,894 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T16:15:51,895 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34435 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T16:15:51,895 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HStore(1541): 63f43c63344ece03d182c42497a352e4/C is initiating minor compaction (all files) 2024-11-23T16:15:51,895 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 63f43c63344ece03d182c42497a352e4/C in TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4. 2024-11-23T16:15:51,895 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/C/3d648d05e92148d3a0e663b21f08e739, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/C/cabd0470aaaa494b8278d295b608ab28, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/C/7a7b449f55c84b91a8a38c77e259b635] into tmpdir=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/.tmp, totalSize=33.6 K 2024-11-23T16:15:51,896 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3d648d05e92148d3a0e663b21f08e739, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=180, earliestPutTs=1732378547241 2024-11-23T16:15:51,896 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.Compactor(225): Compacting cabd0470aaaa494b8278d295b608ab28, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=189, earliestPutTs=1732378548437 2024-11-23T16:15:51,896 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7a7b449f55c84b91a8a38c77e259b635, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=202, earliestPutTs=1732378550626 2024-11-23T16:15:51,897 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 63f43c63344ece03d182c42497a352e4/B of 63f43c63344ece03d182c42497a352e4 into c469a19218cb41a5b69e98708c12c425(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T16:15:51,897 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 63f43c63344ece03d182c42497a352e4: 2024-11-23T16:15:51,897 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., storeName=63f43c63344ece03d182c42497a352e4/B, priority=13, startTime=1732378551472; duration=0sec 2024-11-23T16:15:51,897 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T16:15:51,897 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 63f43c63344ece03d182c42497a352e4:B 2024-11-23T16:15:51,898 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=236 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/.tmp/A/5e891158ee0a4643a9bfb4c8def3d58c 2024-11-23T16:15:51,900 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:51,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39764 deadline: 1732378611888, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:51,901 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:51,901 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:51,901 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:51,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39752 deadline: 1732378611888, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:51,901 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:15:51,901 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:51,901 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:51,901 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:15:51,903 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 63f43c63344ece03d182c42497a352e4#C#compaction#389 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T16:15:51,903 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/.tmp/C/b71d529847614da0bc5d647651606101 is 50, key is test_row_0/C:col10/1732378550626/Put/seqid=0 2024-11-23T16:15:51,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/.tmp/B/16f97a0665064d5dab18689711840cb8 is 50, key is test_row_0/B:col10/1732378550716/Put/seqid=0 2024-11-23T16:15:51,910 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742288_1464 (size=12629) 2024-11-23T16:15:51,913 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:51,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39720 deadline: 1732378611898, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:51,913 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:51,913 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:51,913 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:15:51,914 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742289_1465 (size=12151) 2024-11-23T16:15:51,977 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:51,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39740 deadline: 1732378611968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:51,978 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:51,978 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:51,978 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:15:51,987 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:51,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39788 deadline: 1732378611977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:51,987 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:51,987 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:51,987 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:15:52,198 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:52,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39740 deadline: 1732378612188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:52,199 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:52,199 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:52,199 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:15:52,204 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:52,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39788 deadline: 1732378612198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:52,205 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:52,205 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:52,205 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:15:52,315 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/.tmp/C/b71d529847614da0bc5d647651606101 as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/C/b71d529847614da0bc5d647651606101 2024-11-23T16:15:52,315 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=236 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/.tmp/B/16f97a0665064d5dab18689711840cb8 2024-11-23T16:15:52,319 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 63f43c63344ece03d182c42497a352e4/C of 63f43c63344ece03d182c42497a352e4 into b71d529847614da0bc5d647651606101(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T16:15:52,319 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 63f43c63344ece03d182c42497a352e4: 2024-11-23T16:15:52,319 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., storeName=63f43c63344ece03d182c42497a352e4/C, priority=13, startTime=1732378551472; duration=0sec 2024-11-23T16:15:52,319 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T16:15:52,319 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 63f43c63344ece03d182c42497a352e4:C 2024-11-23T16:15:52,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/.tmp/C/a44d44c2a1db4af98aeede55cb9899c5 is 50, key is test_row_0/C:col10/1732378550716/Put/seqid=0 2024-11-23T16:15:52,333 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742290_1466 (size=12151) 2024-11-23T16:15:52,516 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:52,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39740 deadline: 1732378612507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:52,516 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:52,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39788 deadline: 1732378612508, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:52,516 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:52,516 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:52,516 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:15:52,516 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:52,516 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:52,516 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:15:52,734 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=236 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/.tmp/C/a44d44c2a1db4af98aeede55cb9899c5 2024-11-23T16:15:52,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/.tmp/A/5e891158ee0a4643a9bfb4c8def3d58c as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/A/5e891158ee0a4643a9bfb4c8def3d58c 2024-11-23T16:15:52,741 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/A/5e891158ee0a4643a9bfb4c8def3d58c, entries=150, sequenceid=236, filesize=11.9 K 2024-11-23T16:15:52,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/.tmp/B/16f97a0665064d5dab18689711840cb8 as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/B/16f97a0665064d5dab18689711840cb8 2024-11-23T16:15:52,744 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/B/16f97a0665064d5dab18689711840cb8, entries=150, sequenceid=236, filesize=11.9 K 2024-11-23T16:15:52,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/.tmp/C/a44d44c2a1db4af98aeede55cb9899c5 as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/C/a44d44c2a1db4af98aeede55cb9899c5 2024-11-23T16:15:52,747 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/C/a44d44c2a1db4af98aeede55cb9899c5, entries=150, sequenceid=236, filesize=11.9 K 2024-11-23T16:15:52,748 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~355.64 KB/364176, currentSize=53.67 KB/54960 for 63f43c63344ece03d182c42497a352e4 in 1262ms, sequenceid=236, compaction requested=false 2024-11-23T16:15:52,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2603): Flush status journal for 63f43c63344ece03d182c42497a352e4: 2024-11-23T16:15:52,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4. 2024-11-23T16:15:52,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=120 2024-11-23T16:15:52,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.HMaster(4169): Remote procedure done, pid=120 2024-11-23T16:15:52,751 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=120, resume processing ppid=119 2024-11-23T16:15:52,751 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=120, ppid=119, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0270 sec 2024-11-23T16:15:52,753 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=119, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees in 2.0320 sec 2024-11-23T16:15:52,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-23T16:15:52,869 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-23T16:15:52,869 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-23T16:15:52,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] procedure2.ProcedureExecutor(1139): Stored pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees 2024-11-23T16:15:52,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-23T16:15:52,871 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-23T16:15:52,872 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-23T16:15:52,872 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=122, ppid=121, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-23T16:15:52,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(8855): Flush requested on 63f43c63344ece03d182c42497a352e4 2024-11-23T16:15:52,921 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 63f43c63344ece03d182c42497a352e4 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-23T16:15:52,921 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 63f43c63344ece03d182c42497a352e4, store=A 2024-11-23T16:15:52,921 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-23T16:15:52,921 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 63f43c63344ece03d182c42497a352e4, store=B 2024-11-23T16:15:52,921 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-23T16:15:52,921 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 63f43c63344ece03d182c42497a352e4, store=C 2024-11-23T16:15:52,921 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-23T16:15:52,925 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/.tmp/A/7d041460c2714d1ea1f05f3fdd91bafd is 50, key is test_row_0/A:col10/1732378552919/Put/seqid=0 2024-11-23T16:15:52,932 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742291_1467 (size=14541) 2024-11-23T16:15:52,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-23T16:15:52,983 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:52,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39752 deadline: 1732378612972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:52,983 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:52,983 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:52,983 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:15:52,989 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:52,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39720 deadline: 1732378612975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:52,990 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:52,990 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:52,990 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:15:52,994 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:52,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39764 deadline: 1732378612979, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:52,994 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:52,994 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:52,994 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:15:53,024 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-23T16:15:53,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4. 2024-11-23T16:15:53,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4. as already flushing 2024-11-23T16:15:53,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4. 2024-11-23T16:15:53,024 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 63f43c63344ece03d182c42497a352e4, NAME => 'TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:15:53,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 63f43c63344ece03d182c42497a352e4, NAME => 'TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:15:53,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 63f43c63344ece03d182c42497a352e4, NAME => 'TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 63f43c63344ece03d182c42497a352e4, NAME => 'TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:15:53,038 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:53,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39740 deadline: 1732378613028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:53,039 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:53,039 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:53,039 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:53,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39788 deadline: 1732378613028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:53,039 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:15:53,039 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:53,039 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:53,039 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:15:53,094 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:53,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39752 deadline: 1732378613088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:53,094 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:53,094 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:53,094 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:15:53,107 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:53,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39720 deadline: 1732378613098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:53,108 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:53,108 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:53,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39764 deadline: 1732378613098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:53,108 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:53,108 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:15:53,108 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:53,108 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:53,108 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:15:53,177 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-23T16:15:53,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4. 2024-11-23T16:15:53,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4. as already flushing 2024-11-23T16:15:53,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4. 2024-11-23T16:15:53,177 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 63f43c63344ece03d182c42497a352e4, NAME => 'TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:15:53,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 63f43c63344ece03d182c42497a352e4, NAME => 'TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:15:53,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 63f43c63344ece03d182c42497a352e4, NAME => 'TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 63f43c63344ece03d182c42497a352e4, NAME => 'TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:15:53,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-23T16:15:53,308 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:53,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39752 deadline: 1732378613298, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:53,308 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:53,308 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:53,308 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:15:53,324 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:53,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39720 deadline: 1732378613318, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:53,324 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:53,324 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:53,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39764 deadline: 1732378613318, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:53,324 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:53,324 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:15:53,325 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:53,325 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:53,325 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:15:53,329 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-23T16:15:53,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4. 2024-11-23T16:15:53,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4. as already flushing 2024-11-23T16:15:53,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4. 2024-11-23T16:15:53,330 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 63f43c63344ece03d182c42497a352e4, NAME => 'TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:15:53,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 63f43c63344ece03d182c42497a352e4, NAME => 'TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:15:53,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 63f43c63344ece03d182c42497a352e4, NAME => 'TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 63f43c63344ece03d182c42497a352e4, NAME => 'TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:15:53,333 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=251 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/.tmp/A/7d041460c2714d1ea1f05f3fdd91bafd 2024-11-23T16:15:53,339 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/.tmp/B/04670946fcbe4ae6adc273f7ca906ffe is 50, key is test_row_0/B:col10/1732378552919/Put/seqid=0 2024-11-23T16:15:53,343 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742292_1468 (size=12151) 2024-11-23T16:15:53,482 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-23T16:15:53,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4. 2024-11-23T16:15:53,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4. as already flushing 2024-11-23T16:15:53,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4. 2024-11-23T16:15:53,483 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 63f43c63344ece03d182c42497a352e4, NAME => 'TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:15:53,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 63f43c63344ece03d182c42497a352e4, NAME => 'TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:15:53,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 63f43c63344ece03d182c42497a352e4, NAME => 'TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 63f43c63344ece03d182c42497a352e4, NAME => 'TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:15:53,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-23T16:15:53,625 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:53,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39752 deadline: 1732378613618, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:53,625 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:53,625 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:53,625 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:15:53,635 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-23T16:15:53,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4. 2024-11-23T16:15:53,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4. as already flushing 2024-11-23T16:15:53,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4. 2024-11-23T16:15:53,636 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 63f43c63344ece03d182c42497a352e4, NAME => 'TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:15:53,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 63f43c63344ece03d182c42497a352e4, NAME => 'TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:15:53,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 63f43c63344ece03d182c42497a352e4, NAME => 'TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 63f43c63344ece03d182c42497a352e4, NAME => 'TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:15:53,638 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:53,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39720 deadline: 1732378613628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:53,638 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:53,638 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:53,638 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:53,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39764 deadline: 1732378613628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:53,639 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:15:53,639 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:53,639 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:53,639 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:15:53,743 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=251 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/.tmp/B/04670946fcbe4ae6adc273f7ca906ffe 2024-11-23T16:15:53,749 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/.tmp/C/52f70de1ad22450da38441bcfec9ea4e is 50, key is test_row_0/C:col10/1732378552919/Put/seqid=0 2024-11-23T16:15:53,753 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742293_1469 (size=12151) 2024-11-23T16:15:53,755 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=251 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/.tmp/C/52f70de1ad22450da38441bcfec9ea4e 2024-11-23T16:15:53,758 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/.tmp/A/7d041460c2714d1ea1f05f3fdd91bafd as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/A/7d041460c2714d1ea1f05f3fdd91bafd 2024-11-23T16:15:53,761 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/A/7d041460c2714d1ea1f05f3fdd91bafd, entries=200, sequenceid=251, filesize=14.2 K 2024-11-23T16:15:53,761 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/.tmp/B/04670946fcbe4ae6adc273f7ca906ffe as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/B/04670946fcbe4ae6adc273f7ca906ffe 2024-11-23T16:15:53,764 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/B/04670946fcbe4ae6adc273f7ca906ffe, entries=150, sequenceid=251, filesize=11.9 K 2024-11-23T16:15:53,765 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/.tmp/C/52f70de1ad22450da38441bcfec9ea4e as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/C/52f70de1ad22450da38441bcfec9ea4e 2024-11-23T16:15:53,768 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/C/52f70de1ad22450da38441bcfec9ea4e, entries=150, sequenceid=251, filesize=11.9 K 2024-11-23T16:15:53,769 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 63f43c63344ece03d182c42497a352e4 in 849ms, sequenceid=251, compaction requested=true 2024-11-23T16:15:53,769 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 63f43c63344ece03d182c42497a352e4: 2024-11-23T16:15:53,769 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T16:15:53,769 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 63f43c63344ece03d182c42497a352e4:A, priority=-2147483648, current under compaction store size is 1 2024-11-23T16:15:53,769 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T16:15:53,769 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 63f43c63344ece03d182c42497a352e4:B, priority=-2147483648, current under compaction store size is 2 2024-11-23T16:15:53,769 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T16:15:53,769 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 63f43c63344ece03d182c42497a352e4:C, priority=-2147483648, current under compaction store size is 3 2024-11-23T16:15:53,769 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T16:15:53,769 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T16:15:53,770 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39355 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T16:15:53,770 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36965 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T16:15:53,770 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HStore(1541): 63f43c63344ece03d182c42497a352e4/A is initiating minor compaction (all files) 2024-11-23T16:15:53,770 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HStore(1541): 63f43c63344ece03d182c42497a352e4/B is initiating minor compaction (all files) 2024-11-23T16:15:53,770 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 63f43c63344ece03d182c42497a352e4/A in TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4. 2024-11-23T16:15:53,770 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 63f43c63344ece03d182c42497a352e4/B in TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4. 2024-11-23T16:15:53,770 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/A/b75f3522fd4448da80b8cf120f65f080, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/A/5e891158ee0a4643a9bfb4c8def3d58c, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/A/7d041460c2714d1ea1f05f3fdd91bafd] into tmpdir=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/.tmp, totalSize=38.4 K 2024-11-23T16:15:53,770 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/B/c469a19218cb41a5b69e98708c12c425, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/B/16f97a0665064d5dab18689711840cb8, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/B/04670946fcbe4ae6adc273f7ca906ffe] into tmpdir=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/.tmp, totalSize=36.1 K 2024-11-23T16:15:53,771 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.Compactor(225): Compacting b75f3522fd4448da80b8cf120f65f080, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=202, earliestPutTs=1732378550626 2024-11-23T16:15:53,771 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.Compactor(225): Compacting c469a19218cb41a5b69e98708c12c425, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=202, earliestPutTs=1732378550626 2024-11-23T16:15:53,771 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5e891158ee0a4643a9bfb4c8def3d58c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=236, earliestPutTs=1732378550713 2024-11-23T16:15:53,771 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.Compactor(225): Compacting 16f97a0665064d5dab18689711840cb8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=236, earliestPutTs=1732378550713 2024-11-23T16:15:53,771 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7d041460c2714d1ea1f05f3fdd91bafd, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=251, earliestPutTs=1732378551834 2024-11-23T16:15:53,771 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.Compactor(225): Compacting 04670946fcbe4ae6adc273f7ca906ffe, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=251, earliestPutTs=1732378551834 2024-11-23T16:15:53,777 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 63f43c63344ece03d182c42497a352e4#A#compaction#395 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T16:15:53,777 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/.tmp/A/f9fcba2216d74f96b2d6a5b3350db9c6 is 50, key is test_row_0/A:col10/1732378552919/Put/seqid=0 2024-11-23T16:15:53,780 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 63f43c63344ece03d182c42497a352e4#B#compaction#396 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T16:15:53,780 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/.tmp/B/98d7df745342456997b51fd6577931d8 is 50, key is test_row_0/B:col10/1732378552919/Put/seqid=0 2024-11-23T16:15:53,781 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742294_1470 (size=12765) 2024-11-23T16:15:53,783 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742295_1471 (size=12765) 2024-11-23T16:15:53,788 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-23T16:15:53,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4. 2024-11-23T16:15:53,788 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2902): Flushing 63f43c63344ece03d182c42497a352e4 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-23T16:15:53,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 63f43c63344ece03d182c42497a352e4, store=A 2024-11-23T16:15:53,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-23T16:15:53,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 63f43c63344ece03d182c42497a352e4, store=B 2024-11-23T16:15:53,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-23T16:15:53,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 63f43c63344ece03d182c42497a352e4, store=C 2024-11-23T16:15:53,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-23T16:15:53,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/.tmp/A/1726c374e7104d88bb522f416de35e4b is 50, key is test_row_0/A:col10/1732378552978/Put/seqid=0 2024-11-23T16:15:53,795 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742296_1472 (size=12301) 2024-11-23T16:15:54,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-23T16:15:54,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(8855): Flush requested on 63f43c63344ece03d182c42497a352e4 2024-11-23T16:15:54,052 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4. as already flushing 2024-11-23T16:15:54,091 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:54,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39740 deadline: 1732378614087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:54,092 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:54,092 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:54,092 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:54,092 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:15:54,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39788 deadline: 1732378614088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:54,092 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:54,092 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:54,092 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:15:54,145 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:54,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39752 deadline: 1732378614138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:54,145 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:54,145 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:54,145 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:15:54,158 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:54,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39720 deadline: 1732378614148, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:54,159 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:54,159 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:54,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39764 deadline: 1732378614148, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:54,159 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:54,159 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:15:54,159 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:54,159 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:54,159 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:15:54,185 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/.tmp/A/f9fcba2216d74f96b2d6a5b3350db9c6 as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/A/f9fcba2216d74f96b2d6a5b3350db9c6 2024-11-23T16:15:54,188 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/.tmp/B/98d7df745342456997b51fd6577931d8 as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/B/98d7df745342456997b51fd6577931d8 2024-11-23T16:15:54,189 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 63f43c63344ece03d182c42497a352e4/A of 63f43c63344ece03d182c42497a352e4 into f9fcba2216d74f96b2d6a5b3350db9c6(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T16:15:54,189 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 63f43c63344ece03d182c42497a352e4: 2024-11-23T16:15:54,189 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., storeName=63f43c63344ece03d182c42497a352e4/A, priority=13, startTime=1732378553769; duration=0sec 2024-11-23T16:15:54,189 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T16:15:54,189 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 63f43c63344ece03d182c42497a352e4:A 2024-11-23T16:15:54,189 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T16:15:54,190 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36931 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T16:15:54,190 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HStore(1541): 63f43c63344ece03d182c42497a352e4/C is initiating minor compaction (all files) 2024-11-23T16:15:54,190 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 63f43c63344ece03d182c42497a352e4/C in TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4. 2024-11-23T16:15:54,190 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/C/b71d529847614da0bc5d647651606101, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/C/a44d44c2a1db4af98aeede55cb9899c5, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/C/52f70de1ad22450da38441bcfec9ea4e] into tmpdir=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/.tmp, totalSize=36.1 K 2024-11-23T16:15:54,191 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.Compactor(225): Compacting b71d529847614da0bc5d647651606101, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=202, earliestPutTs=1732378550626 2024-11-23T16:15:54,191 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 63f43c63344ece03d182c42497a352e4/B of 63f43c63344ece03d182c42497a352e4 into 98d7df745342456997b51fd6577931d8(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T16:15:54,191 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 63f43c63344ece03d182c42497a352e4: 2024-11-23T16:15:54,191 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., storeName=63f43c63344ece03d182c42497a352e4/B, priority=13, startTime=1732378553769; duration=0sec 2024-11-23T16:15:54,191 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T16:15:54,191 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 63f43c63344ece03d182c42497a352e4:B 2024-11-23T16:15:54,192 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.Compactor(225): Compacting a44d44c2a1db4af98aeede55cb9899c5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=236, earliestPutTs=1732378550713 2024-11-23T16:15:54,192 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.Compactor(225): Compacting 52f70de1ad22450da38441bcfec9ea4e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=251, earliestPutTs=1732378551834 2024-11-23T16:15:54,195 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=271 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/.tmp/A/1726c374e7104d88bb522f416de35e4b 2024-11-23T16:15:54,198 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 63f43c63344ece03d182c42497a352e4#C#compaction#398 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T16:15:54,199 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/.tmp/C/f451d23b7f304d1cac2cb312a38829d0 is 50, key is test_row_0/C:col10/1732378552919/Put/seqid=0 2024-11-23T16:15:54,200 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:54,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39740 deadline: 1732378614198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:54,201 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:54,201 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:54,201 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:54,201 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:15:54,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39788 deadline: 1732378614198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:54,201 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:54,201 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:54,201 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:15:54,211 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742297_1473 (size=12731) 2024-11-23T16:15:54,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/.tmp/B/11b0e8319b5842f39f82614d5d487b21 is 50, key is test_row_0/B:col10/1732378552978/Put/seqid=0 2024-11-23T16:15:54,216 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742298_1474 (size=12301) 2024-11-23T16:15:54,217 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=271 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/.tmp/B/11b0e8319b5842f39f82614d5d487b21 2024-11-23T16:15:54,222 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/.tmp/C/f451d23b7f304d1cac2cb312a38829d0 as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/C/f451d23b7f304d1cac2cb312a38829d0 2024-11-23T16:15:54,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/.tmp/C/25280f7dea6c4d6eb15a2b71955f5c20 is 50, key is test_row_0/C:col10/1732378552978/Put/seqid=0 2024-11-23T16:15:54,227 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 63f43c63344ece03d182c42497a352e4/C of 63f43c63344ece03d182c42497a352e4 into f451d23b7f304d1cac2cb312a38829d0(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T16:15:54,227 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 63f43c63344ece03d182c42497a352e4: 2024-11-23T16:15:54,227 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., storeName=63f43c63344ece03d182c42497a352e4/C, priority=13, startTime=1732378553769; duration=0sec 2024-11-23T16:15:54,227 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T16:15:54,227 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 63f43c63344ece03d182c42497a352e4:C 2024-11-23T16:15:54,232 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742299_1475 (size=12301) 2024-11-23T16:15:54,416 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:54,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39740 deadline: 1732378614407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:54,417 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:54,417 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:54,417 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:54,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39788 deadline: 1732378614408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:54,417 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:15:54,417 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:54,417 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:54,417 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:15:54,632 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=271 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/.tmp/C/25280f7dea6c4d6eb15a2b71955f5c20 2024-11-23T16:15:54,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/.tmp/A/1726c374e7104d88bb522f416de35e4b as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/A/1726c374e7104d88bb522f416de35e4b 2024-11-23T16:15:54,640 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/A/1726c374e7104d88bb522f416de35e4b, entries=150, sequenceid=271, filesize=12.0 K 2024-11-23T16:15:54,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/.tmp/B/11b0e8319b5842f39f82614d5d487b21 as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/B/11b0e8319b5842f39f82614d5d487b21 2024-11-23T16:15:54,643 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/B/11b0e8319b5842f39f82614d5d487b21, entries=150, sequenceid=271, filesize=12.0 K 2024-11-23T16:15:54,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/.tmp/C/25280f7dea6c4d6eb15a2b71955f5c20 as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/C/25280f7dea6c4d6eb15a2b71955f5c20 2024-11-23T16:15:54,647 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/C/25280f7dea6c4d6eb15a2b71955f5c20, entries=150, sequenceid=271, filesize=12.0 K 2024-11-23T16:15:54,648 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 63f43c63344ece03d182c42497a352e4 in 860ms, sequenceid=271, compaction requested=false 2024-11-23T16:15:54,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2603): Flush status journal for 63f43c63344ece03d182c42497a352e4: 2024-11-23T16:15:54,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4. 2024-11-23T16:15:54,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=122 2024-11-23T16:15:54,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.HMaster(4169): Remote procedure done, pid=122 2024-11-23T16:15:54,651 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=122, resume processing ppid=121 2024-11-23T16:15:54,651 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=122, ppid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7770 sec 2024-11-23T16:15:54,652 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees in 1.7820 sec 2024-11-23T16:15:54,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(8855): Flush requested on 63f43c63344ece03d182c42497a352e4 2024-11-23T16:15:54,731 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 63f43c63344ece03d182c42497a352e4 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-23T16:15:54,731 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 63f43c63344ece03d182c42497a352e4, store=A 2024-11-23T16:15:54,731 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-23T16:15:54,731 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 63f43c63344ece03d182c42497a352e4, store=B 2024-11-23T16:15:54,731 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-23T16:15:54,731 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 63f43c63344ece03d182c42497a352e4, store=C 2024-11-23T16:15:54,731 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-23T16:15:54,735 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/.tmp/A/e4e7bc9d208c47ddb66188e391225790 is 50, key is test_row_0/A:col10/1732378554086/Put/seqid=0 2024-11-23T16:15:54,739 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742300_1476 (size=14741) 2024-11-23T16:15:54,827 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:54,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39740 deadline: 1732378614812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:54,827 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:54,827 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:54,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39788 deadline: 1732378614813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:54,827 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:54,827 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:15:54,827 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:54,827 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:54,827 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:15:54,943 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:54,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39740 deadline: 1732378614938, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:54,943 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:54,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39788 deadline: 1732378614938, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:54,943 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:54,943 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:54,943 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:15:54,943 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:54,943 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:54,944 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:15:55,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-23T16:15:55,018 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-23T16:15:55,018 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-23T16:15:55,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] procedure2.ProcedureExecutor(1139): Stored pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees 2024-11-23T16:15:55,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-23T16:15:55,020 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-23T16:15:55,020 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-23T16:15:55,020 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=124, ppid=123, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-23T16:15:55,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-23T16:15:55,139 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=287 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/.tmp/A/e4e7bc9d208c47ddb66188e391225790 2024-11-23T16:15:55,146 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/.tmp/B/9367c72ebcf149f0b22d94aaaa687317 is 50, key is test_row_0/B:col10/1732378554086/Put/seqid=0 2024-11-23T16:15:55,149 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742301_1477 (size=12301) 2024-11-23T16:15:55,154 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:55,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39740 deadline: 1732378615148, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:55,154 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:55,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39788 deadline: 1732378615148, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:55,154 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:55,154 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:55,154 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:15:55,155 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:55,155 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:55,155 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:15:55,167 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:55,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39752 deadline: 1732378615158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:55,167 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 1a7c5470b472,44715,1732378420375 for 'test_row_1' in 63f43c63344ece03d182c42497a352e4 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2194 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-23T16:15:55,168 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:55,168 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:55,168 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:15:55,172 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:55,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39764 deadline: 1732378615167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:55,172 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-23T16:15:55,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4. 2024-11-23T16:15:55,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4. as already flushing 2024-11-23T16:15:55,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4. 2024-11-23T16:15:55,173 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 63f43c63344ece03d182c42497a352e4, NAME => 'TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:15:55,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 63f43c63344ece03d182c42497a352e4, NAME => 'TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:15:55,173 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 1a7c5470b472,44715,1732378420375 for 'test_row_0' in 63f43c63344ece03d182c42497a352e4 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2193 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-23T16:15:55,173 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:55,173 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:55,173 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:15:55,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 63f43c63344ece03d182c42497a352e4, NAME => 'TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 63f43c63344ece03d182c42497a352e4, NAME => 'TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:15:55,185 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:55,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39720 deadline: 1732378615178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:55,185 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 1a7c5470b472,44715,1732378420375 for 'test_row_1' in 63f43c63344ece03d182c42497a352e4 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2210 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-23T16:15:55,186 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:55,186 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:55,186 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:15:55,324 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-23T16:15:55,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4. 2024-11-23T16:15:55,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4. as already flushing 2024-11-23T16:15:55,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4. 2024-11-23T16:15:55,325 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 63f43c63344ece03d182c42497a352e4, NAME => 'TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:15:55,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 63f43c63344ece03d182c42497a352e4, NAME => 'TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:15:55,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 63f43c63344ece03d182c42497a352e4, NAME => 'TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 63f43c63344ece03d182c42497a352e4, NAME => 'TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:15:55,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-23T16:15:55,465 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:55,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39740 deadline: 1732378615458, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:55,465 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:55,465 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:55,466 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:15:55,477 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-23T16:15:55,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4. 2024-11-23T16:15:55,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4. as already flushing 2024-11-23T16:15:55,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4. 2024-11-23T16:15:55,477 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 63f43c63344ece03d182c42497a352e4, NAME => 'TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:15:55,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 63f43c63344ece03d182c42497a352e4, NAME => 'TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:15:55,478 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:55,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 63f43c63344ece03d182c42497a352e4, NAME => 'TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 63f43c63344ece03d182c42497a352e4, NAME => 'TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:15:55,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39788 deadline: 1732378615468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:55,479 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:55,479 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:55,479 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:15:55,550 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=287 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/.tmp/B/9367c72ebcf149f0b22d94aaaa687317 2024-11-23T16:15:55,556 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/.tmp/C/6f359f53a70943088ebd5269b37fae5b is 50, key is test_row_0/C:col10/1732378554086/Put/seqid=0 2024-11-23T16:15:55,562 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742302_1478 (size=12301) 2024-11-23T16:15:55,630 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-23T16:15:55,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4. 2024-11-23T16:15:55,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4. as already flushing 2024-11-23T16:15:55,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4. 2024-11-23T16:15:55,631 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 63f43c63344ece03d182c42497a352e4, NAME => 'TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:15:55,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 63f43c63344ece03d182c42497a352e4, NAME => 'TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:15:55,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 63f43c63344ece03d182c42497a352e4, NAME => 'TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 63f43c63344ece03d182c42497a352e4, NAME => 'TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:15:55,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-23T16:15:55,783 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-23T16:15:55,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4. 2024-11-23T16:15:55,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4. as already flushing 2024-11-23T16:15:55,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4. 2024-11-23T16:15:55,784 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 63f43c63344ece03d182c42497a352e4, NAME => 'TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:15:55,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 63f43c63344ece03d182c42497a352e4, NAME => 'TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:15:55,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 63f43c63344ece03d182c42497a352e4, NAME => 'TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 63f43c63344ece03d182c42497a352e4, NAME => 'TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:15:55,936 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-23T16:15:55,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4. 2024-11-23T16:15:55,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4. as already flushing 2024-11-23T16:15:55,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4. 2024-11-23T16:15:55,937 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 63f43c63344ece03d182c42497a352e4, NAME => 'TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:15:55,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 63f43c63344ece03d182c42497a352e4, NAME => 'TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:15:55,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 63f43c63344ece03d182c42497a352e4, NAME => 'TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 63f43c63344ece03d182c42497a352e4, NAME => 'TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:15:55,962 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=287 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/.tmp/C/6f359f53a70943088ebd5269b37fae5b 2024-11-23T16:15:55,966 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/.tmp/A/e4e7bc9d208c47ddb66188e391225790 as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/A/e4e7bc9d208c47ddb66188e391225790 2024-11-23T16:15:55,969 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/A/e4e7bc9d208c47ddb66188e391225790, entries=200, sequenceid=287, filesize=14.4 K 2024-11-23T16:15:55,970 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/.tmp/B/9367c72ebcf149f0b22d94aaaa687317 as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/B/9367c72ebcf149f0b22d94aaaa687317 2024-11-23T16:15:55,973 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/B/9367c72ebcf149f0b22d94aaaa687317, entries=150, sequenceid=287, filesize=12.0 K 2024-11-23T16:15:55,973 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/.tmp/C/6f359f53a70943088ebd5269b37fae5b as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/C/6f359f53a70943088ebd5269b37fae5b 2024-11-23T16:15:55,976 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/C/6f359f53a70943088ebd5269b37fae5b, entries=150, sequenceid=287, filesize=12.0 K 2024-11-23T16:15:55,977 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 63f43c63344ece03d182c42497a352e4 in 1246ms, sequenceid=287, compaction requested=true 2024-11-23T16:15:55,977 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 63f43c63344ece03d182c42497a352e4: 2024-11-23T16:15:55,977 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T16:15:55,977 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 63f43c63344ece03d182c42497a352e4:A, priority=-2147483648, current under compaction store size is 1 2024-11-23T16:15:55,977 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T16:15:55,977 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 63f43c63344ece03d182c42497a352e4:B, priority=-2147483648, current under compaction store size is 2 2024-11-23T16:15:55,977 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T16:15:55,977 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 63f43c63344ece03d182c42497a352e4:C, priority=-2147483648, current under compaction store size is 3 2024-11-23T16:15:55,977 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T16:15:55,977 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T16:15:55,978 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39807 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T16:15:55,978 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HStore(1541): 63f43c63344ece03d182c42497a352e4/A is initiating minor compaction (all files) 2024-11-23T16:15:55,978 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 63f43c63344ece03d182c42497a352e4/A in TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4. 2024-11-23T16:15:55,978 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/A/f9fcba2216d74f96b2d6a5b3350db9c6, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/A/1726c374e7104d88bb522f416de35e4b, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/A/e4e7bc9d208c47ddb66188e391225790] into tmpdir=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/.tmp, totalSize=38.9 K 2024-11-23T16:15:55,978 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37367 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T16:15:55,978 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HStore(1541): 63f43c63344ece03d182c42497a352e4/B is initiating minor compaction (all files) 2024-11-23T16:15:55,978 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 63f43c63344ece03d182c42497a352e4/B in TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4. 2024-11-23T16:15:55,978 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/B/98d7df745342456997b51fd6577931d8, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/B/11b0e8319b5842f39f82614d5d487b21, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/B/9367c72ebcf149f0b22d94aaaa687317] into tmpdir=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/.tmp, totalSize=36.5 K 2024-11-23T16:15:55,978 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.Compactor(225): Compacting f9fcba2216d74f96b2d6a5b3350db9c6, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=251, earliestPutTs=1732378551834 2024-11-23T16:15:55,979 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.Compactor(225): Compacting 98d7df745342456997b51fd6577931d8, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=251, earliestPutTs=1732378551834 2024-11-23T16:15:55,979 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1726c374e7104d88bb522f416de35e4b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=271, earliestPutTs=1732378552970 2024-11-23T16:15:55,979 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.Compactor(225): Compacting 11b0e8319b5842f39f82614d5d487b21, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=271, earliestPutTs=1732378552970 2024-11-23T16:15:55,979 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.Compactor(225): Compacting e4e7bc9d208c47ddb66188e391225790, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=287, earliestPutTs=1732378554067 2024-11-23T16:15:55,979 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.Compactor(225): Compacting 9367c72ebcf149f0b22d94aaaa687317, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=287, earliestPutTs=1732378554067 2024-11-23T16:15:55,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(8855): Flush requested on 63f43c63344ece03d182c42497a352e4 2024-11-23T16:15:55,981 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 63f43c63344ece03d182c42497a352e4 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-23T16:15:55,981 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 63f43c63344ece03d182c42497a352e4, store=A 2024-11-23T16:15:55,981 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-23T16:15:55,981 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 63f43c63344ece03d182c42497a352e4, store=B 2024-11-23T16:15:55,981 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-23T16:15:55,981 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 63f43c63344ece03d182c42497a352e4, store=C 2024-11-23T16:15:55,981 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-23T16:15:55,986 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/.tmp/A/0f3755b1e5c44a7a83a249f6e1b53a29 is 50, key is test_row_0/A:col10/1732378554811/Put/seqid=0 2024-11-23T16:15:55,986 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 63f43c63344ece03d182c42497a352e4#B#compaction#406 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-23T16:15:55,987 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 63f43c63344ece03d182c42497a352e4#A#compaction#405 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T16:15:55,987 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/.tmp/B/cb58dbba890d432580f79ee6b2c151be is 50, key is test_row_0/B:col10/1732378554086/Put/seqid=0 2024-11-23T16:15:55,987 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/.tmp/A/9ffada2e725445a3aec3b7ce1e781113 is 50, key is test_row_0/A:col10/1732378554086/Put/seqid=0 2024-11-23T16:15:55,991 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742303_1479 (size=14741) 2024-11-23T16:15:55,993 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742304_1480 (size=13017) 2024-11-23T16:15:55,994 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=307 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/.tmp/A/0f3755b1e5c44a7a83a249f6e1b53a29 2024-11-23T16:15:55,997 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742305_1481 (size=13017) 2024-11-23T16:15:55,999 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/.tmp/B/d9927a7f610644f3a489fa4407a34f8f is 50, key is test_row_0/B:col10/1732378554811/Put/seqid=0 2024-11-23T16:15:56,001 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/.tmp/A/9ffada2e725445a3aec3b7ce1e781113 as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/A/9ffada2e725445a3aec3b7ce1e781113 2024-11-23T16:15:56,006 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742306_1482 (size=12301) 2024-11-23T16:15:56,009 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 63f43c63344ece03d182c42497a352e4/A of 63f43c63344ece03d182c42497a352e4 into 9ffada2e725445a3aec3b7ce1e781113(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T16:15:56,009 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 63f43c63344ece03d182c42497a352e4: 2024-11-23T16:15:56,009 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., storeName=63f43c63344ece03d182c42497a352e4/A, priority=13, startTime=1732378555977; duration=0sec 2024-11-23T16:15:56,009 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T16:15:56,009 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 63f43c63344ece03d182c42497a352e4:A 2024-11-23T16:15:56,009 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T16:15:56,010 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37333 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T16:15:56,010 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HStore(1541): 63f43c63344ece03d182c42497a352e4/C is initiating minor compaction (all files) 2024-11-23T16:15:56,010 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 63f43c63344ece03d182c42497a352e4/C in TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4. 2024-11-23T16:15:56,010 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/C/f451d23b7f304d1cac2cb312a38829d0, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/C/25280f7dea6c4d6eb15a2b71955f5c20, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/C/6f359f53a70943088ebd5269b37fae5b] into tmpdir=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/.tmp, totalSize=36.5 K 2024-11-23T16:15:56,010 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.Compactor(225): Compacting f451d23b7f304d1cac2cb312a38829d0, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=251, earliestPutTs=1732378551834 2024-11-23T16:15:56,011 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.Compactor(225): Compacting 25280f7dea6c4d6eb15a2b71955f5c20, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=271, earliestPutTs=1732378552970 2024-11-23T16:15:56,011 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6f359f53a70943088ebd5269b37fae5b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=287, earliestPutTs=1732378554067 2024-11-23T16:15:56,017 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 63f43c63344ece03d182c42497a352e4#C#compaction#408 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T16:15:56,017 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/.tmp/C/661d7c4f70b64c00a8c8580e7bd0c276 is 50, key is test_row_0/C:col10/1732378554086/Put/seqid=0 2024-11-23T16:15:56,020 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742307_1483 (size=12983) 2024-11-23T16:15:56,024 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/.tmp/C/661d7c4f70b64c00a8c8580e7bd0c276 as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/C/661d7c4f70b64c00a8c8580e7bd0c276 2024-11-23T16:15:56,027 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 63f43c63344ece03d182c42497a352e4/C of 63f43c63344ece03d182c42497a352e4 into 661d7c4f70b64c00a8c8580e7bd0c276(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T16:15:56,027 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 63f43c63344ece03d182c42497a352e4: 2024-11-23T16:15:56,027 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., storeName=63f43c63344ece03d182c42497a352e4/C, priority=13, startTime=1732378555977; duration=0sec 2024-11-23T16:15:56,027 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T16:15:56,027 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 63f43c63344ece03d182c42497a352e4:C 2024-11-23T16:15:56,029 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:56,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39788 deadline: 1732378616017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:56,029 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:56,029 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:56,029 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:15:56,031 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:56,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39740 deadline: 1732378616029, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:56,032 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:56,032 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:56,032 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:15:56,089 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-23T16:15:56,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4. 2024-11-23T16:15:56,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4. as already flushing 2024-11-23T16:15:56,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4. 2024-11-23T16:15:56,090 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 63f43c63344ece03d182c42497a352e4, NAME => 'TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:15:56,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 63f43c63344ece03d182c42497a352e4, NAME => 'TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:15:56,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 63f43c63344ece03d182c42497a352e4, NAME => 'TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 63f43c63344ece03d182c42497a352e4, NAME => 'TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:15:56,145 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:56,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39788 deadline: 1732378616138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:56,145 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:56,146 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:56,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39740 deadline: 1732378616138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:56,146 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:56,146 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:15:56,146 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:56,146 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:56,146 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:15:56,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-23T16:15:56,242 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-23T16:15:56,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4. 2024-11-23T16:15:56,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4. as already flushing 2024-11-23T16:15:56,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4. 2024-11-23T16:15:56,243 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 63f43c63344ece03d182c42497a352e4, NAME => 'TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:15:56,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 63f43c63344ece03d182c42497a352e4, NAME => 'TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:15:56,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 63f43c63344ece03d182c42497a352e4, NAME => 'TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 63f43c63344ece03d182c42497a352e4, NAME => 'TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:15:56,355 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:56,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39788 deadline: 1732378616348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:56,356 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:56,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39740 deadline: 1732378616348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:56,356 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:56,356 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:56,356 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:15:56,356 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:56,356 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:56,356 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:15:56,395 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-23T16:15:56,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4. 2024-11-23T16:15:56,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4. as already flushing 2024-11-23T16:15:56,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4. 2024-11-23T16:15:56,396 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 63f43c63344ece03d182c42497a352e4, NAME => 'TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:15:56,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 63f43c63344ece03d182c42497a352e4, NAME => 'TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:15:56,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 63f43c63344ece03d182c42497a352e4, NAME => 'TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 63f43c63344ece03d182c42497a352e4, NAME => 'TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:15:56,399 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/.tmp/B/cb58dbba890d432580f79ee6b2c151be as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/B/cb58dbba890d432580f79ee6b2c151be 2024-11-23T16:15:56,403 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 63f43c63344ece03d182c42497a352e4/B of 63f43c63344ece03d182c42497a352e4 into cb58dbba890d432580f79ee6b2c151be(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T16:15:56,403 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 63f43c63344ece03d182c42497a352e4: 2024-11-23T16:15:56,403 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., storeName=63f43c63344ece03d182c42497a352e4/B, priority=13, startTime=1732378555977; duration=0sec 2024-11-23T16:15:56,403 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T16:15:56,403 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 63f43c63344ece03d182c42497a352e4:B 2024-11-23T16:15:56,406 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=307 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/.tmp/B/d9927a7f610644f3a489fa4407a34f8f 2024-11-23T16:15:56,413 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/.tmp/C/724b560fe3894a9aa2ec63a6e4f1873e is 50, key is test_row_0/C:col10/1732378554811/Put/seqid=0 2024-11-23T16:15:56,416 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742308_1484 (size=12301) 2024-11-23T16:15:56,549 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-23T16:15:56,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4. 2024-11-23T16:15:56,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4. as already flushing 2024-11-23T16:15:56,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4. 2024-11-23T16:15:56,549 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 63f43c63344ece03d182c42497a352e4, NAME => 'TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:15:56,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 63f43c63344ece03d182c42497a352e4, NAME => 'TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:15:56,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 63f43c63344ece03d182c42497a352e4, NAME => 'TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 63f43c63344ece03d182c42497a352e4, NAME => 'TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:15:56,665 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:56,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39740 deadline: 1732378616658, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:56,665 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:56,665 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:56,665 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:15:56,676 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:56,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39788 deadline: 1732378616668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:56,677 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:56,677 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:56,677 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:15:56,702 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-23T16:15:56,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4. 2024-11-23T16:15:56,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4. as already flushing 2024-11-23T16:15:56,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4. 2024-11-23T16:15:56,702 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 63f43c63344ece03d182c42497a352e4, NAME => 'TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:15:56,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 63f43c63344ece03d182c42497a352e4, NAME => 'TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:15:56,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 63f43c63344ece03d182c42497a352e4, NAME => 'TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 63f43c63344ece03d182c42497a352e4, NAME => 'TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:15:56,816 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=307 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/.tmp/C/724b560fe3894a9aa2ec63a6e4f1873e 2024-11-23T16:15:56,820 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/.tmp/A/0f3755b1e5c44a7a83a249f6e1b53a29 as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/A/0f3755b1e5c44a7a83a249f6e1b53a29 2024-11-23T16:15:56,823 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/A/0f3755b1e5c44a7a83a249f6e1b53a29, entries=200, sequenceid=307, filesize=14.4 K 2024-11-23T16:15:56,824 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/.tmp/B/d9927a7f610644f3a489fa4407a34f8f as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/B/d9927a7f610644f3a489fa4407a34f8f 2024-11-23T16:15:56,827 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/B/d9927a7f610644f3a489fa4407a34f8f, entries=150, sequenceid=307, filesize=12.0 K 2024-11-23T16:15:56,828 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/.tmp/C/724b560fe3894a9aa2ec63a6e4f1873e as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/C/724b560fe3894a9aa2ec63a6e4f1873e 2024-11-23T16:15:56,832 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/C/724b560fe3894a9aa2ec63a6e4f1873e, entries=150, sequenceid=307, filesize=12.0 K 2024-11-23T16:15:56,834 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 63f43c63344ece03d182c42497a352e4 in 853ms, sequenceid=307, compaction requested=false 2024-11-23T16:15:56,834 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 63f43c63344ece03d182c42497a352e4: 2024-11-23T16:15:56,855 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-23T16:15:56,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4. 2024-11-23T16:15:56,856 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2902): Flushing 63f43c63344ece03d182c42497a352e4 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-23T16:15:56,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 63f43c63344ece03d182c42497a352e4, store=A 2024-11-23T16:15:56,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-23T16:15:56,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 63f43c63344ece03d182c42497a352e4, store=B 2024-11-23T16:15:56,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-23T16:15:56,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 63f43c63344ece03d182c42497a352e4, store=C 2024-11-23T16:15:56,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-23T16:15:56,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/.tmp/A/7e8a7c2c98464f0f9d746be30d7608dd is 50, key is test_row_0/A:col10/1732378556028/Put/seqid=0 2024-11-23T16:15:56,870 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742309_1485 (size=12301) 2024-11-23T16:15:57,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-23T16:15:57,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(8855): Flush requested on 63f43c63344ece03d182c42497a352e4 2024-11-23T16:15:57,181 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4. as already flushing 2024-11-23T16:15:57,227 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:57,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39720 deadline: 1732378617213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:57,227 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:57,227 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:57,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39764 deadline: 1732378617214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:57,227 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:57,227 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:15:57,227 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:57,228 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:57,228 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:57,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39752 deadline: 1732378617215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:57,228 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:15:57,228 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:57,228 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:57,228 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:15:57,228 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:57,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39788 deadline: 1732378617216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:57,229 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:57,229 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:57,229 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:57,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39740 deadline: 1732378617227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:57,229 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:15:57,229 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:57,229 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:57,229 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:15:57,270 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=322 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/.tmp/A/7e8a7c2c98464f0f9d746be30d7608dd 2024-11-23T16:15:57,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/.tmp/B/3e2a12b146f840b9b0590000da83d051 is 50, key is test_row_0/B:col10/1732378556028/Put/seqid=0 2024-11-23T16:15:57,280 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742310_1486 (size=12301) 2024-11-23T16:15:57,281 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=322 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/.tmp/B/3e2a12b146f840b9b0590000da83d051 2024-11-23T16:15:57,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/.tmp/C/dc979853a15d4ca1b8a617e6db03db34 is 50, key is test_row_0/C:col10/1732378556028/Put/seqid=0 2024-11-23T16:15:57,290 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742311_1487 (size=12301) 2024-11-23T16:15:57,351 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:57,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39720 deadline: 1732378617338, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:57,352 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:57,352 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:57,352 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:57,352 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:15:57,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39764 deadline: 1732378617338, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:57,352 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:57,352 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:57,353 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:57,353 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:15:57,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39752 deadline: 1732378617338, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:57,353 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:57,353 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:57,353 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:57,353 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:15:57,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39788 deadline: 1732378617338, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:57,353 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:57,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39740 deadline: 1732378617338, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:57,353 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:57,353 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:57,353 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:15:57,354 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:57,354 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:57,354 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:15:57,565 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:57,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39720 deadline: 1732378617558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:57,566 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:57,566 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:57,566 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:57,566 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:15:57,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39764 deadline: 1732378617558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:57,566 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:57,566 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:57,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39752 deadline: 1732378617558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:57,566 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:57,566 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:15:57,566 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:57,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39788 deadline: 1732378617558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:57,566 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:57,566 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:57,567 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:15:57,567 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:57,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39740 deadline: 1732378617559, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:57,567 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:57,567 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:57,567 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:15:57,567 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:57,567 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:57,567 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:15:57,692 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=322 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/.tmp/C/dc979853a15d4ca1b8a617e6db03db34 2024-11-23T16:15:57,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/.tmp/A/7e8a7c2c98464f0f9d746be30d7608dd as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/A/7e8a7c2c98464f0f9d746be30d7608dd 2024-11-23T16:15:57,699 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/A/7e8a7c2c98464f0f9d746be30d7608dd, entries=150, sequenceid=322, filesize=12.0 K 2024-11-23T16:15:57,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/.tmp/B/3e2a12b146f840b9b0590000da83d051 as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/B/3e2a12b146f840b9b0590000da83d051 2024-11-23T16:15:57,703 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/B/3e2a12b146f840b9b0590000da83d051, entries=150, sequenceid=322, filesize=12.0 K 2024-11-23T16:15:57,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/.tmp/C/dc979853a15d4ca1b8a617e6db03db34 as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/C/dc979853a15d4ca1b8a617e6db03db34 2024-11-23T16:15:57,707 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/C/dc979853a15d4ca1b8a617e6db03db34, entries=150, sequenceid=322, filesize=12.0 K 2024-11-23T16:15:57,708 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 63f43c63344ece03d182c42497a352e4 in 852ms, sequenceid=322, compaction requested=true 2024-11-23T16:15:57,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2603): Flush status journal for 63f43c63344ece03d182c42497a352e4: 2024-11-23T16:15:57,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4. 2024-11-23T16:15:57,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=124 2024-11-23T16:15:57,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.HMaster(4169): Remote procedure done, pid=124 2024-11-23T16:15:57,711 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=124, resume processing ppid=123 2024-11-23T16:15:57,711 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=124, ppid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6890 sec 2024-11-23T16:15:57,713 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees in 2.6940 sec 2024-11-23T16:15:57,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(8855): Flush requested on 63f43c63344ece03d182c42497a352e4 2024-11-23T16:15:57,876 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 63f43c63344ece03d182c42497a352e4 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-11-23T16:15:57,877 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 63f43c63344ece03d182c42497a352e4, store=A 2024-11-23T16:15:57,877 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-23T16:15:57,877 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 63f43c63344ece03d182c42497a352e4, store=B 2024-11-23T16:15:57,877 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-23T16:15:57,877 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 63f43c63344ece03d182c42497a352e4, store=C 2024-11-23T16:15:57,877 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-23T16:15:57,882 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/.tmp/A/388c96d9a56a4f998767a7a865f289dd is 50, key is test_row_0/A:col10/1732378557874/Put/seqid=0 2024-11-23T16:15:57,885 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742312_1488 (size=14741) 2024-11-23T16:15:57,886 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=343 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/.tmp/A/388c96d9a56a4f998767a7a865f289dd 2024-11-23T16:15:57,892 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/.tmp/B/afa1aae79c1041e28935011142718324 is 50, key is test_row_0/B:col10/1732378557874/Put/seqid=0 2024-11-23T16:15:57,896 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742313_1489 (size=12301) 2024-11-23T16:15:57,896 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=343 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/.tmp/B/afa1aae79c1041e28935011142718324 2024-11-23T16:15:57,900 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:57,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39788 deadline: 1732378617891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:57,900 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:57,900 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:57,900 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:15:57,903 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/.tmp/C/8c38b95c143f41d3b49faa2e97b8b89c is 50, key is test_row_0/C:col10/1732378557874/Put/seqid=0 2024-11-23T16:15:57,906 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:57,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39740 deadline: 1732378617892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:57,907 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:57,907 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:57,907 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:57,907 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:15:57,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39764 deadline: 1732378617894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:57,907 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:57,907 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:57,907 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:15:57,907 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:57,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39752 deadline: 1732378617895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:57,908 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:57,908 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:57,908 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:15:57,912 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:57,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39720 deadline: 1732378617896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:57,912 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:57,912 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:57,912 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:15:57,913 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742314_1490 (size=12301) 2024-11-23T16:15:57,914 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=343 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/.tmp/C/8c38b95c143f41d3b49faa2e97b8b89c 2024-11-23T16:15:57,917 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/.tmp/A/388c96d9a56a4f998767a7a865f289dd as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/A/388c96d9a56a4f998767a7a865f289dd 2024-11-23T16:15:57,920 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/A/388c96d9a56a4f998767a7a865f289dd, entries=200, sequenceid=343, filesize=14.4 K 2024-11-23T16:15:57,921 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/.tmp/B/afa1aae79c1041e28935011142718324 as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/B/afa1aae79c1041e28935011142718324 2024-11-23T16:15:57,924 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/B/afa1aae79c1041e28935011142718324, entries=150, sequenceid=343, filesize=12.0 K 2024-11-23T16:15:57,925 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/.tmp/C/8c38b95c143f41d3b49faa2e97b8b89c as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/C/8c38b95c143f41d3b49faa2e97b8b89c 2024-11-23T16:15:57,927 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/C/8c38b95c143f41d3b49faa2e97b8b89c, entries=150, sequenceid=343, filesize=12.0 K 2024-11-23T16:15:57,928 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=53.67 KB/54960 for 63f43c63344ece03d182c42497a352e4 in 52ms, sequenceid=343, compaction requested=true 2024-11-23T16:15:57,928 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 63f43c63344ece03d182c42497a352e4: 2024-11-23T16:15:57,928 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 63f43c63344ece03d182c42497a352e4:A, priority=-2147483648, current under compaction store size is 1 2024-11-23T16:15:57,928 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T16:15:57,928 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-23T16:15:57,928 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 63f43c63344ece03d182c42497a352e4:B, priority=-2147483648, current under compaction store size is 2 2024-11-23T16:15:57,928 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T16:15:57,928 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 63f43c63344ece03d182c42497a352e4:C, priority=-2147483648, current under compaction store size is 3 2024-11-23T16:15:57,928 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T16:15:57,928 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-23T16:15:57,931 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49920 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-23T16:15:57,932 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HStore(1541): 63f43c63344ece03d182c42497a352e4/B is initiating minor compaction (all files) 2024-11-23T16:15:57,932 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 63f43c63344ece03d182c42497a352e4/B in TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4. 2024-11-23T16:15:57,932 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/B/cb58dbba890d432580f79ee6b2c151be, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/B/d9927a7f610644f3a489fa4407a34f8f, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/B/3e2a12b146f840b9b0590000da83d051, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/B/afa1aae79c1041e28935011142718324] into tmpdir=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/.tmp, totalSize=48.8 K 2024-11-23T16:15:57,932 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.Compactor(225): Compacting cb58dbba890d432580f79ee6b2c151be, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=287, earliestPutTs=1732378554067 2024-11-23T16:15:57,933 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.Compactor(225): Compacting d9927a7f610644f3a489fa4407a34f8f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=307, earliestPutTs=1732378554782 2024-11-23T16:15:57,933 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 54800 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-23T16:15:57,933 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HStore(1541): 63f43c63344ece03d182c42497a352e4/A is initiating minor compaction (all files) 2024-11-23T16:15:57,933 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 63f43c63344ece03d182c42497a352e4/A in TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4. 2024-11-23T16:15:57,933 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/A/9ffada2e725445a3aec3b7ce1e781113, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/A/0f3755b1e5c44a7a83a249f6e1b53a29, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/A/7e8a7c2c98464f0f9d746be30d7608dd, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/A/388c96d9a56a4f998767a7a865f289dd] into tmpdir=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/.tmp, totalSize=53.5 K 2024-11-23T16:15:57,933 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.Compactor(225): Compacting 3e2a12b146f840b9b0590000da83d051, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=322, earliestPutTs=1732378556001 2024-11-23T16:15:57,933 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9ffada2e725445a3aec3b7ce1e781113, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=287, earliestPutTs=1732378554067 2024-11-23T16:15:57,933 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.Compactor(225): Compacting afa1aae79c1041e28935011142718324, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=343, earliestPutTs=1732378557212 2024-11-23T16:15:57,934 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0f3755b1e5c44a7a83a249f6e1b53a29, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=307, earliestPutTs=1732378554782 2024-11-23T16:15:57,934 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7e8a7c2c98464f0f9d746be30d7608dd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=322, earliestPutTs=1732378556001 2024-11-23T16:15:57,934 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.Compactor(225): Compacting 388c96d9a56a4f998767a7a865f289dd, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=343, earliestPutTs=1732378557212 2024-11-23T16:15:57,943 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 63f43c63344ece03d182c42497a352e4#B#compaction#416 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-23T16:15:57,943 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/.tmp/B/ec230b48dfb345928edadfdc2fc3a778 is 50, key is test_row_0/B:col10/1732378557874/Put/seqid=0 2024-11-23T16:15:57,944 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 63f43c63344ece03d182c42497a352e4#A#compaction#417 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T16:15:57,945 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/.tmp/A/d5eca5bad3a547c8b6ecfd1a48b1f464 is 50, key is test_row_0/A:col10/1732378557874/Put/seqid=0 2024-11-23T16:15:57,951 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742315_1491 (size=13153) 2024-11-23T16:15:57,952 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742316_1492 (size=13153) 2024-11-23T16:15:58,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(8855): Flush requested on 63f43c63344ece03d182c42497a352e4 2024-11-23T16:15:58,016 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 63f43c63344ece03d182c42497a352e4 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-23T16:15:58,016 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 63f43c63344ece03d182c42497a352e4, store=A 2024-11-23T16:15:58,016 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-23T16:15:58,016 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 63f43c63344ece03d182c42497a352e4, store=B 2024-11-23T16:15:58,016 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-23T16:15:58,016 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 63f43c63344ece03d182c42497a352e4, store=C 2024-11-23T16:15:58,016 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-23T16:15:58,020 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/.tmp/A/fb6c56f5b0774eddbcddb5f874ccbd43 is 50, key is test_row_0/A:col10/1732378557893/Put/seqid=0 2024-11-23T16:15:58,023 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742317_1493 (size=14741) 2024-11-23T16:15:58,069 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1a7c5470b472:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11854, totalHeapSize=27176, min timestamp=1732378558024, max timestamp=1732378558067Num uniques 859; 2024-11-23T16:15:58,069 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1a7c5470b472:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=11854, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-23T16:15:58,071 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1a7c5470b472:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11854, totalHeapSize=27176, min timestamp=1732378558024, max timestamp=1732378558067Num uniques 859; 2024-11-23T16:15:58,071 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1a7c5470b472:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=11854, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-23T16:15:58,071 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1a7c5470b472:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11854, totalHeapSize=27176, min timestamp=1732378558024, max timestamp=1732378558067Num uniques 859; 2024-11-23T16:15:58,071 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1a7c5470b472:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=11854, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-23T16:15:58,116 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:58,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39788 deadline: 1732378618097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:58,116 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:58,116 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:58,116 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:58,116 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:58,116 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:15:58,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39740 deadline: 1732378618098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:58,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39764 deadline: 1732378618101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:58,117 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:58,117 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:58,117 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:15:58,117 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:58,117 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:58,117 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:58,117 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:15:58,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39720 deadline: 1732378618109, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:58,117 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:58,117 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:58,117 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:15:58,131 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:58,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39752 deadline: 1732378618110, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:58,131 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:58,131 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:58,131 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:15:58,239 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:58,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39788 deadline: 1732378618228, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:58,240 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:58,240 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:58,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39740 deadline: 1732378618228, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:58,240 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:58,240 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:15:58,240 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:58,240 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:58,240 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:58,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39764 deadline: 1732378618228, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:58,240 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:15:58,241 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:58,241 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:58,241 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:58,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39720 deadline: 1732378618228, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:58,241 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:15:58,241 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:58,241 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:58,241 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:15:58,252 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:58,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39752 deadline: 1732378618237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:58,252 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:58,252 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:58,252 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:15:58,356 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/.tmp/A/d5eca5bad3a547c8b6ecfd1a48b1f464 as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/A/d5eca5bad3a547c8b6ecfd1a48b1f464 2024-11-23T16:15:58,356 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/.tmp/B/ec230b48dfb345928edadfdc2fc3a778 as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/B/ec230b48dfb345928edadfdc2fc3a778 2024-11-23T16:15:58,366 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 63f43c63344ece03d182c42497a352e4/A of 63f43c63344ece03d182c42497a352e4 into d5eca5bad3a547c8b6ecfd1a48b1f464(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T16:15:58,366 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 63f43c63344ece03d182c42497a352e4/B of 63f43c63344ece03d182c42497a352e4 into ec230b48dfb345928edadfdc2fc3a778(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T16:15:58,366 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 63f43c63344ece03d182c42497a352e4: 2024-11-23T16:15:58,366 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 63f43c63344ece03d182c42497a352e4: 2024-11-23T16:15:58,366 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., storeName=63f43c63344ece03d182c42497a352e4/B, priority=12, startTime=1732378557928; duration=0sec 2024-11-23T16:15:58,366 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., storeName=63f43c63344ece03d182c42497a352e4/A, priority=12, startTime=1732378557928; duration=0sec 2024-11-23T16:15:58,366 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T16:15:58,366 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 63f43c63344ece03d182c42497a352e4:B 2024-11-23T16:15:58,366 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T16:15:58,366 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 63f43c63344ece03d182c42497a352e4:A 2024-11-23T16:15:58,366 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-23T16:15:58,367 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49886 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-23T16:15:58,367 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HStore(1541): 63f43c63344ece03d182c42497a352e4/C is initiating minor compaction (all files) 2024-11-23T16:15:58,367 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 63f43c63344ece03d182c42497a352e4/C in TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4. 2024-11-23T16:15:58,367 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/C/661d7c4f70b64c00a8c8580e7bd0c276, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/C/724b560fe3894a9aa2ec63a6e4f1873e, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/C/dc979853a15d4ca1b8a617e6db03db34, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/C/8c38b95c143f41d3b49faa2e97b8b89c] into tmpdir=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/.tmp, totalSize=48.7 K 2024-11-23T16:15:58,368 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.Compactor(225): Compacting 661d7c4f70b64c00a8c8580e7bd0c276, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=287, earliestPutTs=1732378554067 2024-11-23T16:15:58,368 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.Compactor(225): Compacting 724b560fe3894a9aa2ec63a6e4f1873e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=307, earliestPutTs=1732378554782 2024-11-23T16:15:58,368 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.Compactor(225): Compacting dc979853a15d4ca1b8a617e6db03db34, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=322, earliestPutTs=1732378556001 2024-11-23T16:15:58,368 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.Compactor(225): Compacting 8c38b95c143f41d3b49faa2e97b8b89c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=343, earliestPutTs=1732378557212 2024-11-23T16:15:58,376 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 63f43c63344ece03d182c42497a352e4#C#compaction#419 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T16:15:58,377 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/.tmp/C/00618d75c3a94f639cbdb10f360e8f5b is 50, key is test_row_0/C:col10/1732378557874/Put/seqid=0 2024-11-23T16:15:58,383 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742318_1494 (size=13119) 2024-11-23T16:15:58,423 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=355 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/.tmp/A/fb6c56f5b0774eddbcddb5f874ccbd43 2024-11-23T16:15:58,429 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/.tmp/B/41c359a4030c4a93a374b36decace292 is 50, key is test_row_0/B:col10/1732378557893/Put/seqid=0 2024-11-23T16:15:58,432 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742319_1495 (size=12301) 2024-11-23T16:15:58,456 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:58,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39788 deadline: 1732378618448, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:58,457 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:58,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39740 deadline: 1732378618448, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:58,457 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:58,457 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:58,457 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:15:58,457 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:58,457 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:58,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39764 deadline: 1732378618448, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:58,457 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:58,457 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:15:58,457 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:58,457 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:58,457 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:58,457 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:15:58,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39720 deadline: 1732378618449, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:58,458 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:58,458 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:58,458 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:15:58,469 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:58,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39752 deadline: 1732378618458, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:58,470 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:58,470 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:58,470 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:15:58,780 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:58,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39788 deadline: 1732378618768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:58,780 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:58,780 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:58,780 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:58,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39740 deadline: 1732378618768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:58,781 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:15:58,781 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:58,781 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:58,781 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:58,781 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:15:58,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39764 deadline: 1732378618768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:58,781 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:58,781 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:58,781 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:58,781 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:15:58,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39720 deadline: 1732378618768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:58,781 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:58,781 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:58,781 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:15:58,781 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:58,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39752 deadline: 1732378618778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:58,782 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:58,782 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:58,782 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:15:58,787 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/.tmp/C/00618d75c3a94f639cbdb10f360e8f5b as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/C/00618d75c3a94f639cbdb10f360e8f5b 2024-11-23T16:15:58,791 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 63f43c63344ece03d182c42497a352e4/C of 63f43c63344ece03d182c42497a352e4 into 00618d75c3a94f639cbdb10f360e8f5b(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T16:15:58,791 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 63f43c63344ece03d182c42497a352e4: 2024-11-23T16:15:58,791 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., storeName=63f43c63344ece03d182c42497a352e4/C, priority=12, startTime=1732378557928; duration=0sec 2024-11-23T16:15:58,791 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T16:15:58,791 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 63f43c63344ece03d182c42497a352e4:C 2024-11-23T16:15:58,833 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=355 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/.tmp/B/41c359a4030c4a93a374b36decace292 2024-11-23T16:15:58,839 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/.tmp/C/6f812fd0d0bb43d495ffb1049646cfa3 is 50, key is test_row_0/C:col10/1732378557893/Put/seqid=0 2024-11-23T16:15:58,843 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742320_1496 (size=12301) 2024-11-23T16:15:59,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-23T16:15:59,178 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-23T16:15:59,178 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-23T16:15:59,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] procedure2.ProcedureExecutor(1139): Stored pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees 2024-11-23T16:15:59,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-23T16:15:59,180 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-23T16:15:59,181 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-23T16:15:59,181 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=126, ppid=125, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-23T16:15:59,243 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=355 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/.tmp/C/6f812fd0d0bb43d495ffb1049646cfa3 2024-11-23T16:15:59,247 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/.tmp/A/fb6c56f5b0774eddbcddb5f874ccbd43 as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/A/fb6c56f5b0774eddbcddb5f874ccbd43 2024-11-23T16:15:59,250 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/A/fb6c56f5b0774eddbcddb5f874ccbd43, entries=200, sequenceid=355, filesize=14.4 K 2024-11-23T16:15:59,251 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/.tmp/B/41c359a4030c4a93a374b36decace292 as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/B/41c359a4030c4a93a374b36decace292 2024-11-23T16:15:59,256 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/B/41c359a4030c4a93a374b36decace292, entries=150, sequenceid=355, filesize=12.0 K 2024-11-23T16:15:59,257 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/.tmp/C/6f812fd0d0bb43d495ffb1049646cfa3 as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/C/6f812fd0d0bb43d495ffb1049646cfa3 2024-11-23T16:15:59,261 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/C/6f812fd0d0bb43d495ffb1049646cfa3, entries=150, sequenceid=355, filesize=12.0 K 2024-11-23T16:15:59,262 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=120.76 KB/123660 for 63f43c63344ece03d182c42497a352e4 in 1247ms, sequenceid=355, compaction requested=false 2024-11-23T16:15:59,262 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 63f43c63344ece03d182c42497a352e4: 2024-11-23T16:15:59,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-23T16:15:59,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(8855): Flush requested on 63f43c63344ece03d182c42497a352e4 2024-11-23T16:15:59,294 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 63f43c63344ece03d182c42497a352e4 3/3 column families, dataSize=127.47 KB heapSize=355.69 KB 2024-11-23T16:15:59,294 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 63f43c63344ece03d182c42497a352e4, store=A 2024-11-23T16:15:59,294 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-23T16:15:59,294 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 63f43c63344ece03d182c42497a352e4, store=B 2024-11-23T16:15:59,294 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-23T16:15:59,294 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 63f43c63344ece03d182c42497a352e4, store=C 2024-11-23T16:15:59,294 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-23T16:15:59,299 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/.tmp/A/c4c89b2c3eb9412685e407c00147a596 is 50, key is test_row_0/A:col10/1732378558096/Put/seqid=0 2024-11-23T16:15:59,302 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742321_1497 (size=14741) 2024-11-23T16:15:59,313 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:59,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39788 deadline: 1732378619299, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:59,313 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:59,313 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:59,313 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:15:59,313 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:59,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39740 deadline: 1732378619312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:59,314 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:59,314 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:59,314 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:59,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39764 deadline: 1732378619313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:59,314 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:15:59,314 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:59,314 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:59,314 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:15:59,325 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:59,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39720 deadline: 1732378619314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:59,325 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:59,325 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:59,325 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:15:59,328 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:59,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39752 deadline: 1732378619314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:59,329 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:59,329 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:59,329 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:15:59,333 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-23T16:15:59,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4. 2024-11-23T16:15:59,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4. as already flushing 2024-11-23T16:15:59,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4. 2024-11-23T16:15:59,333 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 63f43c63344ece03d182c42497a352e4, NAME => 'TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:15:59,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 63f43c63344ece03d182c42497a352e4, NAME => 'TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:15:59,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 63f43c63344ece03d182c42497a352e4, NAME => 'TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 63f43c63344ece03d182c42497a352e4, NAME => 'TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:15:59,430 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:59,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39788 deadline: 1732378619418, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:59,431 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:59,431 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:59,431 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:59,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39740 deadline: 1732378619418, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:59,431 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:15:59,431 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:59,431 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:59,431 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:59,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39764 deadline: 1732378619418, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:59,431 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:15:59,432 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:59,432 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:59,432 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:15:59,434 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:59,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39720 deadline: 1732378619427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:59,434 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:59,434 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:59,434 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:15:59,449 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:59,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39752 deadline: 1732378619438, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:59,450 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:59,450 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:59,450 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:15:59,485 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-23T16:15:59,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4. 2024-11-23T16:15:59,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4. as already flushing 2024-11-23T16:15:59,486 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4. 2024-11-23T16:15:59,486 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 63f43c63344ece03d182c42497a352e4, NAME => 'TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:15:59,486 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 63f43c63344ece03d182c42497a352e4, NAME => 'TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:15:59,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 63f43c63344ece03d182c42497a352e4, NAME => 'TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 63f43c63344ece03d182c42497a352e4, NAME => 'TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:15:59,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-23T16:15:59,638 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-23T16:15:59,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4. 2024-11-23T16:15:59,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4. as already flushing 2024-11-23T16:15:59,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4. 2024-11-23T16:15:59,639 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 63f43c63344ece03d182c42497a352e4, NAME => 'TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:15:59,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 63f43c63344ece03d182c42497a352e4, NAME => 'TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:15:59,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 63f43c63344ece03d182c42497a352e4, NAME => 'TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 63f43c63344ece03d182c42497a352e4, NAME => 'TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:15:59,652 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:59,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39788 deadline: 1732378619638, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:59,652 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:59,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39740 deadline: 1732378619638, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:59,652 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:59,652 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:59,652 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:15:59,652 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:59,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39764 deadline: 1732378619639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:59,652 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:59,652 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:59,652 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:15:59,652 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:59,652 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:59,653 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:59,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39720 deadline: 1732378619639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:59,653 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:15:59,653 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:59,653 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:59,653 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:15:59,668 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:59,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39752 deadline: 1732378619658, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:59,668 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:59,669 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:59,669 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:15:59,703 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=392 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/.tmp/A/c4c89b2c3eb9412685e407c00147a596 2024-11-23T16:15:59,710 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/.tmp/B/ed9715959c184b87b8e6361f91db3f45 is 50, key is test_row_0/B:col10/1732378558096/Put/seqid=0 2024-11-23T16:15:59,713 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742322_1498 (size=12301) 2024-11-23T16:15:59,714 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=392 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/.tmp/B/ed9715959c184b87b8e6361f91db3f45 2024-11-23T16:15:59,721 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/.tmp/C/aa76e434783242f3a6d8d24020a71920 is 50, key is test_row_0/C:col10/1732378558096/Put/seqid=0 2024-11-23T16:15:59,727 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742323_1499 (size=12301) 2024-11-23T16:15:59,791 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-23T16:15:59,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4. 2024-11-23T16:15:59,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4. as already flushing 2024-11-23T16:15:59,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4. 2024-11-23T16:15:59,792 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 63f43c63344ece03d182c42497a352e4, NAME => 'TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:15:59,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 63f43c63344ece03d182c42497a352e4, NAME => 'TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:15:59,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 63f43c63344ece03d182c42497a352e4, NAME => 'TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 63f43c63344ece03d182c42497a352e4, NAME => 'TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:15:59,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-23T16:15:59,944 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-23T16:15:59,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4. 2024-11-23T16:15:59,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4. as already flushing 2024-11-23T16:15:59,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4. 2024-11-23T16:15:59,944 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 63f43c63344ece03d182c42497a352e4, NAME => 'TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:15:59,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 63f43c63344ece03d182c42497a352e4, NAME => 'TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:15:59,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 63f43c63344ece03d182c42497a352e4, NAME => 'TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 63f43c63344ece03d182c42497a352e4, NAME => 'TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:15:59,972 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:59,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39788 deadline: 1732378619958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:59,972 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:59,972 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:59,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39740 deadline: 1732378619958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:59,972 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:59,972 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:15:59,973 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:59,973 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:59,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39764 deadline: 1732378619958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:59,973 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:59,973 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:15:59,973 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:59,973 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:59,973 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:59,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39720 deadline: 1732378619958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:59,973 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:15:59,973 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:59,973 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:59,973 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:15:59,989 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:15:59,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39752 deadline: 1732378619978, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:15:59,989 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:59,990 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:15:59,990 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:16:00,097 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-23T16:16:00,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4. 2024-11-23T16:16:00,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4. as already flushing 2024-11-23T16:16:00,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4. 2024-11-23T16:16:00,097 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 63f43c63344ece03d182c42497a352e4, NAME => 'TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:16:00,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 63f43c63344ece03d182c42497a352e4, NAME => 'TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:16:00,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 63f43c63344ece03d182c42497a352e4, NAME => 'TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 63f43c63344ece03d182c42497a352e4, NAME => 'TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:16:00,128 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=392 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/.tmp/C/aa76e434783242f3a6d8d24020a71920 2024-11-23T16:16:00,132 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/.tmp/A/c4c89b2c3eb9412685e407c00147a596 as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/A/c4c89b2c3eb9412685e407c00147a596 2024-11-23T16:16:00,135 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/A/c4c89b2c3eb9412685e407c00147a596, entries=200, sequenceid=392, filesize=14.4 K 2024-11-23T16:16:00,135 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/.tmp/B/ed9715959c184b87b8e6361f91db3f45 as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/B/ed9715959c184b87b8e6361f91db3f45 2024-11-23T16:16:00,139 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/B/ed9715959c184b87b8e6361f91db3f45, entries=150, sequenceid=392, filesize=12.0 K 2024-11-23T16:16:00,139 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/.tmp/C/aa76e434783242f3a6d8d24020a71920 as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/C/aa76e434783242f3a6d8d24020a71920 2024-11-23T16:16:00,142 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/C/aa76e434783242f3a6d8d24020a71920, entries=150, sequenceid=392, filesize=12.0 K 2024-11-23T16:16:00,143 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~355.64 KB/364176, currentSize=60.38 KB/61830 for 63f43c63344ece03d182c42497a352e4 in 848ms, sequenceid=392, compaction requested=true 2024-11-23T16:16:00,143 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 63f43c63344ece03d182c42497a352e4: 2024-11-23T16:16:00,143 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 63f43c63344ece03d182c42497a352e4:A, priority=-2147483648, current under compaction store size is 1 2024-11-23T16:16:00,143 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T16:16:00,143 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T16:16:00,143 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 63f43c63344ece03d182c42497a352e4:B, priority=-2147483648, current under compaction store size is 2 2024-11-23T16:16:00,143 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T16:16:00,143 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 63f43c63344ece03d182c42497a352e4:C, priority=-2147483648, current under compaction store size is 3 2024-11-23T16:16:00,143 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T16:16:00,143 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T16:16:00,144 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42635 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T16:16:00,144 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HStore(1541): 63f43c63344ece03d182c42497a352e4/A is initiating minor compaction (all files) 2024-11-23T16:16:00,144 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 63f43c63344ece03d182c42497a352e4/A in TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4. 2024-11-23T16:16:00,144 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/A/d5eca5bad3a547c8b6ecfd1a48b1f464, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/A/fb6c56f5b0774eddbcddb5f874ccbd43, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/A/c4c89b2c3eb9412685e407c00147a596] into tmpdir=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/.tmp, totalSize=41.6 K 2024-11-23T16:16:00,144 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T16:16:00,144 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.Compactor(225): Compacting d5eca5bad3a547c8b6ecfd1a48b1f464, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=343, earliestPutTs=1732378557212 2024-11-23T16:16:00,144 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HStore(1541): 63f43c63344ece03d182c42497a352e4/B is initiating minor compaction (all files) 2024-11-23T16:16:00,144 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 63f43c63344ece03d182c42497a352e4/B in TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4. 2024-11-23T16:16:00,144 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/B/ec230b48dfb345928edadfdc2fc3a778, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/B/41c359a4030c4a93a374b36decace292, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/B/ed9715959c184b87b8e6361f91db3f45] into tmpdir=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/.tmp, totalSize=36.9 K 2024-11-23T16:16:00,144 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.Compactor(225): Compacting fb6c56f5b0774eddbcddb5f874ccbd43, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=355, earliestPutTs=1732378557885 2024-11-23T16:16:00,145 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.Compactor(225): Compacting ec230b48dfb345928edadfdc2fc3a778, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=343, earliestPutTs=1732378557212 2024-11-23T16:16:00,145 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.Compactor(225): Compacting c4c89b2c3eb9412685e407c00147a596, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=392, earliestPutTs=1732378558094 2024-11-23T16:16:00,145 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.Compactor(225): Compacting 41c359a4030c4a93a374b36decace292, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=355, earliestPutTs=1732378557885 2024-11-23T16:16:00,145 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.Compactor(225): Compacting ed9715959c184b87b8e6361f91db3f45, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=392, earliestPutTs=1732378558094 2024-11-23T16:16:00,153 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 63f43c63344ece03d182c42497a352e4#A#compaction#425 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T16:16:00,154 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/.tmp/A/27ebffaf4ce9489b88ea9bf4ef2669d9 is 50, key is test_row_0/A:col10/1732378558096/Put/seqid=0 2024-11-23T16:16:00,158 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 63f43c63344ece03d182c42497a352e4#B#compaction#426 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T16:16:00,159 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/.tmp/B/ce975a5dd97943a783adf2b915e28562 is 50, key is test_row_0/B:col10/1732378558096/Put/seqid=0 2024-11-23T16:16:00,161 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742324_1500 (size=13255) 2024-11-23T16:16:00,165 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742325_1501 (size=13255) 2024-11-23T16:16:00,250 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-23T16:16:00,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4. 2024-11-23T16:16:00,250 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2902): Flushing 63f43c63344ece03d182c42497a352e4 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-23T16:16:00,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 63f43c63344ece03d182c42497a352e4, store=A 2024-11-23T16:16:00,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-23T16:16:00,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 63f43c63344ece03d182c42497a352e4, store=B 2024-11-23T16:16:00,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-23T16:16:00,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 63f43c63344ece03d182c42497a352e4, store=C 2024-11-23T16:16:00,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-23T16:16:00,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/.tmp/A/c05147b5dcea44c290714de5d7676bf0 is 50, key is test_row_0/A:col10/1732378559297/Put/seqid=0 2024-11-23T16:16:00,257 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742326_1502 (size=12301) 2024-11-23T16:16:00,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-23T16:16:00,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(8855): Flush requested on 63f43c63344ece03d182c42497a352e4 2024-11-23T16:16:00,481 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4. as already flushing 2024-11-23T16:16:00,523 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:16:00,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39752 deadline: 1732378620511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:16:00,524 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:00,524 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:00,524 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:16:00,524 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:16:00,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39764 deadline: 1732378620512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:16:00,525 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:16:00,525 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:00,525 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:00,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39720 deadline: 1732378620513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:16:00,525 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:16:00,525 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:00,525 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:00,525 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:16:00,526 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:16:00,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39740 deadline: 1732378620523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:16:00,526 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:00,526 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:00,526 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:16:00,537 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:16:00,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39788 deadline: 1732378620524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:16:00,537 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:00,537 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:00,537 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:16:00,565 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/.tmp/A/27ebffaf4ce9489b88ea9bf4ef2669d9 as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/A/27ebffaf4ce9489b88ea9bf4ef2669d9 2024-11-23T16:16:00,569 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/.tmp/B/ce975a5dd97943a783adf2b915e28562 as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/B/ce975a5dd97943a783adf2b915e28562 2024-11-23T16:16:00,570 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 63f43c63344ece03d182c42497a352e4/A of 63f43c63344ece03d182c42497a352e4 into 27ebffaf4ce9489b88ea9bf4ef2669d9(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T16:16:00,570 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 63f43c63344ece03d182c42497a352e4: 2024-11-23T16:16:00,570 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., storeName=63f43c63344ece03d182c42497a352e4/A, priority=13, startTime=1732378560143; duration=0sec 2024-11-23T16:16:00,570 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T16:16:00,570 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 63f43c63344ece03d182c42497a352e4:A 2024-11-23T16:16:00,570 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T16:16:00,571 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T16:16:00,571 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HStore(1541): 63f43c63344ece03d182c42497a352e4/C is initiating minor compaction (all files) 2024-11-23T16:16:00,571 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 63f43c63344ece03d182c42497a352e4/C in TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4. 2024-11-23T16:16:00,571 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/C/00618d75c3a94f639cbdb10f360e8f5b, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/C/6f812fd0d0bb43d495ffb1049646cfa3, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/C/aa76e434783242f3a6d8d24020a71920] into tmpdir=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/.tmp, totalSize=36.8 K 2024-11-23T16:16:00,571 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.Compactor(225): Compacting 00618d75c3a94f639cbdb10f360e8f5b, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=343, earliestPutTs=1732378557212 2024-11-23T16:16:00,571 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6f812fd0d0bb43d495ffb1049646cfa3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=355, earliestPutTs=1732378557885 2024-11-23T16:16:00,572 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.Compactor(225): Compacting aa76e434783242f3a6d8d24020a71920, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=392, earliestPutTs=1732378558094 2024-11-23T16:16:00,573 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 63f43c63344ece03d182c42497a352e4/B of 63f43c63344ece03d182c42497a352e4 into ce975a5dd97943a783adf2b915e28562(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T16:16:00,573 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 63f43c63344ece03d182c42497a352e4: 2024-11-23T16:16:00,573 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., storeName=63f43c63344ece03d182c42497a352e4/B, priority=13, startTime=1732378560143; duration=0sec 2024-11-23T16:16:00,573 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T16:16:00,573 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 63f43c63344ece03d182c42497a352e4:B 2024-11-23T16:16:00,579 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 63f43c63344ece03d182c42497a352e4#C#compaction#428 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T16:16:00,579 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/.tmp/C/71386003914b42d683b04091a7efa642 is 50, key is test_row_0/C:col10/1732378558096/Put/seqid=0 2024-11-23T16:16:00,584 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742327_1503 (size=13221) 2024-11-23T16:16:00,633 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:16:00,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39752 deadline: 1732378620628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:16:00,633 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:00,633 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:16:00,633 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:00,633 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:16:00,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39764 deadline: 1732378620628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:16:00,634 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:16:00,634 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:00,634 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:00,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39720 deadline: 1732378620628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:16:00,634 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:16:00,634 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:16:00,634 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:00,634 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:00,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39740 deadline: 1732378620629, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:16:00,634 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:16:00,634 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:00,634 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:00,634 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:16:00,657 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:16:00,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39788 deadline: 1732378620648, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:16:00,658 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:00,658 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:00,658 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:16:00,658 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=404 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/.tmp/A/c05147b5dcea44c290714de5d7676bf0 2024-11-23T16:16:00,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/.tmp/B/f2eff5e552974887aceeda069398a33d is 50, key is test_row_0/B:col10/1732378559297/Put/seqid=0 2024-11-23T16:16:00,668 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742328_1504 (size=12301) 2024-11-23T16:16:00,847 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:16:00,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39752 deadline: 1732378620838, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:16:00,848 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:16:00,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39764 deadline: 1732378620838, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:16:00,848 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:00,848 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:00,848 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:16:00,848 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:16:00,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39720 deadline: 1732378620838, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:16:00,848 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:00,848 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:00,848 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:16:00,848 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:16:00,848 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:00,849 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:00,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39740 deadline: 1732378620839, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:16:00,849 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:16:00,849 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:00,849 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:00,849 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:16:00,873 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:16:00,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39788 deadline: 1732378620868, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:16:00,874 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 , the old value is region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:00,874 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:00,874 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., hostname=1a7c5470b472,44715,1732378420375, seqNum=2 because the exception is null or not the one we care about 2024-11-23T16:16:00,989 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/.tmp/C/71386003914b42d683b04091a7efa642 as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/C/71386003914b42d683b04091a7efa642 2024-11-23T16:16:00,996 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 63f43c63344ece03d182c42497a352e4/C of 63f43c63344ece03d182c42497a352e4 into 71386003914b42d683b04091a7efa642(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T16:16:00,996 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 63f43c63344ece03d182c42497a352e4: 2024-11-23T16:16:00,996 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., storeName=63f43c63344ece03d182c42497a352e4/C, priority=13, startTime=1732378560143; duration=0sec 2024-11-23T16:16:00,996 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T16:16:00,996 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 63f43c63344ece03d182c42497a352e4:C 2024-11-23T16:16:01,069 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=404 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/.tmp/B/f2eff5e552974887aceeda069398a33d 2024-11-23T16:16:01,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/.tmp/C/5ea77280771a4ae69d8821d8e76852a4 is 50, key is test_row_0/C:col10/1732378559297/Put/seqid=0 2024-11-23T16:16:01,086 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742329_1505 (size=12301) 2024-11-23T16:16:01,086 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=404 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/.tmp/C/5ea77280771a4ae69d8821d8e76852a4 2024-11-23T16:16:01,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/.tmp/A/c05147b5dcea44c290714de5d7676bf0 as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/A/c05147b5dcea44c290714de5d7676bf0 2024-11-23T16:16:01,097 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/A/c05147b5dcea44c290714de5d7676bf0, entries=150, sequenceid=404, filesize=12.0 K 2024-11-23T16:16:01,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/.tmp/B/f2eff5e552974887aceeda069398a33d as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/B/f2eff5e552974887aceeda069398a33d 2024-11-23T16:16:01,102 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/B/f2eff5e552974887aceeda069398a33d, entries=150, sequenceid=404, filesize=12.0 K 2024-11-23T16:16:01,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/.tmp/C/5ea77280771a4ae69d8821d8e76852a4 as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/C/5ea77280771a4ae69d8821d8e76852a4 2024-11-23T16:16:01,109 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/C/5ea77280771a4ae69d8821d8e76852a4, entries=150, sequenceid=404, filesize=12.0 K 2024-11-23T16:16:01,110 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 63f43c63344ece03d182c42497a352e4 in 860ms, sequenceid=404, compaction requested=false 2024-11-23T16:16:01,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2603): Flush status journal for 63f43c63344ece03d182c42497a352e4: 2024-11-23T16:16:01,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4. 2024-11-23T16:16:01,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=126 2024-11-23T16:16:01,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.HMaster(4169): Remote procedure done, pid=126 2024-11-23T16:16:01,116 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=126, resume processing ppid=125 2024-11-23T16:16:01,116 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=126, ppid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9330 sec 2024-11-23T16:16:01,118 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees in 1.9390 sec 2024-11-23T16:16:01,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(8855): Flush requested on 63f43c63344ece03d182c42497a352e4 2024-11-23T16:16:01,167 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 63f43c63344ece03d182c42497a352e4 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-11-23T16:16:01,168 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1a7c5470b472:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732378560506, max timestamp=1732378561165Num uniques 859; 2024-11-23T16:16:01,168 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1a7c5470b472:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732378560506, max timestamp=1732378561165Num uniques 859; 2024-11-23T16:16:01,169 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1a7c5470b472:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732378560506, max timestamp=1732378561165Num uniques 859; 2024-11-23T16:16:01,169 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1a7c5470b472:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-23T16:16:01,169 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1a7c5470b472:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-23T16:16:01,169 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1a7c5470b472:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-23T16:16:01,169 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 63f43c63344ece03d182c42497a352e4, store=A 2024-11-23T16:16:01,169 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-23T16:16:01,169 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 63f43c63344ece03d182c42497a352e4, store=B 2024-11-23T16:16:01,169 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-23T16:16:01,169 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 63f43c63344ece03d182c42497a352e4, store=C 2024-11-23T16:16:01,169 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-23T16:16:01,173 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/.tmp/A/072612d169b74c6a884de1fe09d96cca is 50, key is test_row_0/A:col10/1732378560507/Put/seqid=0 2024-11-23T16:16:01,179 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742330_1506 (size=12301) 2024-11-23T16:16:01,214 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1a7c5470b472:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732378561198, max timestamp=1732378561211Num uniques 859; 2024-11-23T16:16:01,214 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1a7c5470b472:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-23T16:16:01,215 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1a7c5470b472:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732378561198, max timestamp=1732378561211Num uniques 859; 2024-11-23T16:16:01,215 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1a7c5470b472:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-23T16:16:01,215 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1a7c5470b472:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732378561198, max timestamp=1732378561211Num uniques 859; 2024-11-23T16:16:01,215 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1a7c5470b472:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-23T16:16:01,229 INFO [Thread-1898 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1898. 2024-11-23T16:16:01,229 DEBUG [Thread-1898 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-23T16:16:01,229 DEBUG [Thread-1898 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T16:16:01,229 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-23T16:16:01,230 DEBUG [Thread-1898 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T16:16:01,230 INFO [Thread-1889 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1889. 2024-11-23T16:16:01,230 DEBUG [Thread-1889 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-23T16:16:01,230 DEBUG [Thread-1889 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T16:16:01,230 DEBUG [Thread-1889 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T16:16:01,230 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-23T16:16:01,231 INFO [Thread-1883 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1883. 2024-11-23T16:16:01,231 DEBUG [Thread-1883 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-23T16:16:01,231 DEBUG [Thread-1883 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T16:16:01,231 DEBUG [Thread-1883 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T16:16:01,231 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-23T16:16:01,232 INFO [Thread-1896 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1896. 2024-11-23T16:16:01,233 DEBUG [Thread-1896 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-23T16:16:01,233 DEBUG [Thread-1896 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T16:16:01,233 DEBUG [Thread-1896 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T16:16:01,233 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-23T16:16:01,233 INFO [Thread-1887 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1887. 2024-11-23T16:16:01,233 DEBUG [Thread-1887 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-23T16:16:01,233 DEBUG [Thread-1887 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T16:16:01,233 DEBUG [Thread-1887 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T16:16:01,233 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-23T16:16:01,233 INFO [Thread-1881 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1881. 2024-11-23T16:16:01,233 DEBUG [Thread-1881 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-23T16:16:01,233 DEBUG [Thread-1881 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T16:16:01,233 DEBUG [Thread-1881 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T16:16:01,233 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-23T16:16:01,234 INFO [Thread-1885 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1885. 2024-11-23T16:16:01,234 DEBUG [Thread-1885 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-23T16:16:01,234 DEBUG [Thread-1885 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T16:16:01,235 DEBUG [Thread-1885 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T16:16:01,235 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-23T16:16:01,242 INFO [Thread-1900 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1900. 2024-11-23T16:16:01,242 DEBUG [Thread-1900 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-23T16:16:01,242 DEBUG [Thread-1900 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T16:16:01,242 DEBUG [Thread-1900 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T16:16:01,242 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-23T16:16:01,251 INFO [Thread-1892 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1892. 2024-11-23T16:16:01,251 DEBUG [Thread-1892 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-23T16:16:01,251 DEBUG [Thread-1892 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T16:16:01,251 DEBUG [Thread-1892 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T16:16:01,251 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-23T16:16:01,252 INFO [Thread-1894 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1894. 2024-11-23T16:16:01,252 DEBUG [Thread-1894 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-23T16:16:01,252 DEBUG [Thread-1894 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T16:16:01,252 DEBUG [Thread-1894 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T16:16:01,252 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-23T16:16:01,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-23T16:16:01,338 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-23T16:16:01,338 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-23T16:16:01,338 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 74 2024-11-23T16:16:01,338 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 81 2024-11-23T16:16:01,338 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 83 2024-11-23T16:16:01,338 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 60 2024-11-23T16:16:01,338 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 65 2024-11-23T16:16:01,338 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-23T16:16:01,338 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-23T16:16:01,338 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1119 2024-11-23T16:16:01,338 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3355 rows 2024-11-23T16:16:01,338 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1117 2024-11-23T16:16:01,338 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3351 rows 2024-11-23T16:16:01,338 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1116 2024-11-23T16:16:01,338 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3345 rows 2024-11-23T16:16:01,338 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1115 2024-11-23T16:16:01,338 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3345 rows 2024-11-23T16:16:01,338 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1104 2024-11-23T16:16:01,338 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3312 rows 2024-11-23T16:16:01,338 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-23T16:16:01,338 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testScanAtomicity(AcidGuaranteesTestBase.java:112) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-23T16:16:01,338 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T16:16:01,339 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T16:16:01,339 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-23T16:16:01,339 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-23T16:16:01,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] procedure2.ProcedureExecutor(1139): Stored pid=127, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-23T16:16:01,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-23T16:16:01,342 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732378561342"}]},"ts":"1732378561342"} 2024-11-23T16:16:01,343 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-23T16:16:01,343 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-23T16:16:01,343 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=128, ppid=127, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-23T16:16:01,344 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=129, ppid=128, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=63f43c63344ece03d182c42497a352e4, UNASSIGN}] 2024-11-23T16:16:01,345 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=129, ppid=128, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=63f43c63344ece03d182c42497a352e4, UNASSIGN 2024-11-23T16:16:01,346 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=129 updating hbase:meta row=63f43c63344ece03d182c42497a352e4, regionState=CLOSING, regionLocation=1a7c5470b472,44715,1732378420375 2024-11-23T16:16:01,347 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=129, ppid=128, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=63f43c63344ece03d182c42497a352e4, UNASSIGN because future has completed 2024-11-23T16:16:01,347 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-23T16:16:01,347 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=130, ppid=129, state=RUNNABLE, hasLock=false; CloseRegionProcedure 63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375}] 2024-11-23T16:16:01,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-23T16:16:01,499 INFO [RS_CLOSE_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] handler.UnassignRegionHandler(122): Close 63f43c63344ece03d182c42497a352e4 2024-11-23T16:16:01,499 DEBUG [RS_CLOSE_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-23T16:16:01,499 DEBUG [RS_CLOSE_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] regionserver.HRegion(1722): Closing 63f43c63344ece03d182c42497a352e4, disabling compactions & flushes 2024-11-23T16:16:01,499 DEBUG [RS_CLOSE_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] regionserver.HRegion(1993): waiting for 0 compactions & cache flush to complete for region TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4. 2024-11-23T16:16:01,579 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=429 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/.tmp/A/072612d169b74c6a884de1fe09d96cca 2024-11-23T16:16:01,585 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/.tmp/B/c11ef502b4d943b19f4bc54c0b9014cb is 50, key is test_row_0/B:col10/1732378560507/Put/seqid=0 2024-11-23T16:16:01,587 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742331_1507 (size=12301) 2024-11-23T16:16:01,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-23T16:16:01,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-23T16:16:01,988 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=429 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/.tmp/B/c11ef502b4d943b19f4bc54c0b9014cb 2024-11-23T16:16:01,993 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/.tmp/C/35832a69ef2540e7b7b0a9a9075f92e9 is 50, key is test_row_0/C:col10/1732378560507/Put/seqid=0 2024-11-23T16:16:01,996 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742332_1508 (size=12301) 2024-11-23T16:16:02,397 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=429 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/.tmp/C/35832a69ef2540e7b7b0a9a9075f92e9 2024-11-23T16:16:02,400 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/.tmp/A/072612d169b74c6a884de1fe09d96cca as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/A/072612d169b74c6a884de1fe09d96cca 2024-11-23T16:16:02,403 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/A/072612d169b74c6a884de1fe09d96cca, entries=150, sequenceid=429, filesize=12.0 K 2024-11-23T16:16:02,404 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/.tmp/B/c11ef502b4d943b19f4bc54c0b9014cb as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/B/c11ef502b4d943b19f4bc54c0b9014cb 2024-11-23T16:16:02,406 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/B/c11ef502b4d943b19f4bc54c0b9014cb, entries=150, sequenceid=429, filesize=12.0 K 2024-11-23T16:16:02,406 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/.tmp/C/35832a69ef2540e7b7b0a9a9075f92e9 as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/C/35832a69ef2540e7b7b0a9a9075f92e9 2024-11-23T16:16:02,409 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/C/35832a69ef2540e7b7b0a9a9075f92e9, entries=150, sequenceid=429, filesize=12.0 K 2024-11-23T16:16:02,410 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~86.11 KB/88176, currentSize=93.93 KB/96180 for 63f43c63344ece03d182c42497a352e4 in 1243ms, sequenceid=429, compaction requested=true 2024-11-23T16:16:02,410 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 63f43c63344ece03d182c42497a352e4: 2024-11-23T16:16:02,410 INFO [RS_CLOSE_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4. 2024-11-23T16:16:02,410 DEBUG [RS_CLOSE_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4. 2024-11-23T16:16:02,410 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 63f43c63344ece03d182c42497a352e4:A, priority=-2147483648, current under compaction store size is 1 2024-11-23T16:16:02,410 DEBUG [RS_CLOSE_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4. after waiting 0 ms 2024-11-23T16:16:02,410 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4. because compaction request was cancelled 2024-11-23T16:16:02,410 DEBUG [RS_CLOSE_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4. 2024-11-23T16:16:02,410 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 63f43c63344ece03d182c42497a352e4:A 2024-11-23T16:16:02,410 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T16:16:02,410 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 63f43c63344ece03d182c42497a352e4:B, priority=-2147483648, current under compaction store size is 1 2024-11-23T16:16:02,410 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4. because compaction request was cancelled 2024-11-23T16:16:02,410 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T16:16:02,410 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 63f43c63344ece03d182c42497a352e4:B 2024-11-23T16:16:02,410 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4. because compaction request was cancelled 2024-11-23T16:16:02,410 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 63f43c63344ece03d182c42497a352e4:C, priority=-2147483648, current under compaction store size is 1 2024-11-23T16:16:02,410 INFO [RS_CLOSE_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] regionserver.HRegion(2902): Flushing 63f43c63344ece03d182c42497a352e4 3/3 column families, dataSize=93.93 KB heapSize=260.77 KB 2024-11-23T16:16:02,410 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 63f43c63344ece03d182c42497a352e4:C 2024-11-23T16:16:02,410 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T16:16:02,410 DEBUG [RS_CLOSE_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 63f43c63344ece03d182c42497a352e4, store=A 2024-11-23T16:16:02,410 DEBUG [RS_CLOSE_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-23T16:16:02,410 DEBUG [RS_CLOSE_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 63f43c63344ece03d182c42497a352e4, store=B 2024-11-23T16:16:02,410 DEBUG [RS_CLOSE_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-23T16:16:02,410 DEBUG [RS_CLOSE_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 63f43c63344ece03d182c42497a352e4, store=C 2024-11-23T16:16:02,410 DEBUG [RS_CLOSE_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-23T16:16:02,413 DEBUG [RS_CLOSE_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/.tmp/A/dfd7a28e34614d8ab4c9f15f4d03a6d3 is 50, key is test_row_0/A:col10/1732378561231/Put/seqid=0 2024-11-23T16:16:02,416 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742333_1509 (size=12301) 2024-11-23T16:16:02,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-23T16:16:02,817 INFO [RS_CLOSE_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=459 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/.tmp/A/dfd7a28e34614d8ab4c9f15f4d03a6d3 2024-11-23T16:16:02,822 DEBUG [RS_CLOSE_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/.tmp/B/f4359489557b4d65b3f287c81b23589a is 50, key is test_row_0/B:col10/1732378561231/Put/seqid=0 2024-11-23T16:16:02,825 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742334_1510 (size=12301) 2024-11-23T16:16:03,225 INFO [RS_CLOSE_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=459 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/.tmp/B/f4359489557b4d65b3f287c81b23589a 2024-11-23T16:16:03,231 DEBUG [RS_CLOSE_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/.tmp/C/c78f575db88345ccb9112bf85711ca14 is 50, key is test_row_0/C:col10/1732378561231/Put/seqid=0 2024-11-23T16:16:03,234 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742335_1511 (size=12301) 2024-11-23T16:16:03,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-23T16:16:03,634 INFO [RS_CLOSE_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=459 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/.tmp/C/c78f575db88345ccb9112bf85711ca14 2024-11-23T16:16:03,638 DEBUG [RS_CLOSE_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/.tmp/A/dfd7a28e34614d8ab4c9f15f4d03a6d3 as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/A/dfd7a28e34614d8ab4c9f15f4d03a6d3 2024-11-23T16:16:03,640 INFO [RS_CLOSE_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/A/dfd7a28e34614d8ab4c9f15f4d03a6d3, entries=150, sequenceid=459, filesize=12.0 K 2024-11-23T16:16:03,641 DEBUG [RS_CLOSE_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/.tmp/B/f4359489557b4d65b3f287c81b23589a as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/B/f4359489557b4d65b3f287c81b23589a 2024-11-23T16:16:03,643 INFO [RS_CLOSE_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/B/f4359489557b4d65b3f287c81b23589a, entries=150, sequenceid=459, filesize=12.0 K 2024-11-23T16:16:03,644 DEBUG [RS_CLOSE_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/.tmp/C/c78f575db88345ccb9112bf85711ca14 as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/C/c78f575db88345ccb9112bf85711ca14 2024-11-23T16:16:03,646 INFO [RS_CLOSE_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/C/c78f575db88345ccb9112bf85711ca14, entries=150, sequenceid=459, filesize=12.0 K 2024-11-23T16:16:03,647 INFO [RS_CLOSE_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~260.72 KB/266976, currentSize=0 B/0 for 63f43c63344ece03d182c42497a352e4 in 1237ms, sequenceid=459, compaction requested=true 2024-11-23T16:16:03,647 DEBUG [StoreCloser-TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/A/e407c533aeae46619bebcd49c9222c75, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/A/1ae48cc8fc0b4836b60eddf3acb17fb5, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/A/c7eb4f5aa1c746b9af5c69ac0701fa0c, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/A/ff1641249c5d4a988529f2e1773cd2d2, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/A/e2101deb540c4868823ae0c48251af1e, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/A/49462c0916b149179a8e28b17cb17d66, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/A/bcd9e3e7285b4b1f8ce668f410f8b772, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/A/83acfcade2b6405e88a009b9937c1f29, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/A/aa62cb4d2fe7489dbe2e53148271e649, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/A/d64612b7e9af45218c8c57a8af10c624, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/A/230b2f3c02c84defb484385b5a644f0e, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/A/3c5ff086e7164122a4757921e19087e8, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/A/72eae2668f414abd857521062e3d9477, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/A/5e3be1e387b94f7c91c829c1071cadc1, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/A/b712b5711ed4432587b0c6c286d24466, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/A/b75f3522fd4448da80b8cf120f65f080, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/A/5e891158ee0a4643a9bfb4c8def3d58c, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/A/7d041460c2714d1ea1f05f3fdd91bafd, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/A/f9fcba2216d74f96b2d6a5b3350db9c6, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/A/1726c374e7104d88bb522f416de35e4b, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/A/e4e7bc9d208c47ddb66188e391225790, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/A/9ffada2e725445a3aec3b7ce1e781113, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/A/0f3755b1e5c44a7a83a249f6e1b53a29, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/A/7e8a7c2c98464f0f9d746be30d7608dd, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/A/388c96d9a56a4f998767a7a865f289dd, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/A/d5eca5bad3a547c8b6ecfd1a48b1f464, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/A/fb6c56f5b0774eddbcddb5f874ccbd43, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/A/c4c89b2c3eb9412685e407c00147a596] to archive 2024-11-23T16:16:03,648 DEBUG [StoreCloser-TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-23T16:16:03,649 DEBUG [StoreCloser-TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/A/e407c533aeae46619bebcd49c9222c75 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/A/e407c533aeae46619bebcd49c9222c75 2024-11-23T16:16:03,650 DEBUG [StoreCloser-TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/A/1ae48cc8fc0b4836b60eddf3acb17fb5 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/A/1ae48cc8fc0b4836b60eddf3acb17fb5 2024-11-23T16:16:03,651 DEBUG [StoreCloser-TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/A/c7eb4f5aa1c746b9af5c69ac0701fa0c to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/A/c7eb4f5aa1c746b9af5c69ac0701fa0c 2024-11-23T16:16:03,651 DEBUG [StoreCloser-TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/A/ff1641249c5d4a988529f2e1773cd2d2 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/A/ff1641249c5d4a988529f2e1773cd2d2 2024-11-23T16:16:03,652 DEBUG [StoreCloser-TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/A/e2101deb540c4868823ae0c48251af1e to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/A/e2101deb540c4868823ae0c48251af1e 2024-11-23T16:16:03,653 DEBUG [StoreCloser-TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/A/49462c0916b149179a8e28b17cb17d66 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/A/49462c0916b149179a8e28b17cb17d66 2024-11-23T16:16:03,654 DEBUG [StoreCloser-TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/A/bcd9e3e7285b4b1f8ce668f410f8b772 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/A/bcd9e3e7285b4b1f8ce668f410f8b772 2024-11-23T16:16:03,655 DEBUG [StoreCloser-TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/A/83acfcade2b6405e88a009b9937c1f29 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/A/83acfcade2b6405e88a009b9937c1f29 2024-11-23T16:16:03,655 DEBUG [StoreCloser-TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/A/aa62cb4d2fe7489dbe2e53148271e649 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/A/aa62cb4d2fe7489dbe2e53148271e649 2024-11-23T16:16:03,656 DEBUG [StoreCloser-TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/A/d64612b7e9af45218c8c57a8af10c624 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/A/d64612b7e9af45218c8c57a8af10c624 2024-11-23T16:16:03,657 DEBUG [StoreCloser-TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/A/230b2f3c02c84defb484385b5a644f0e to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/A/230b2f3c02c84defb484385b5a644f0e 2024-11-23T16:16:03,657 DEBUG [StoreCloser-TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/A/3c5ff086e7164122a4757921e19087e8 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/A/3c5ff086e7164122a4757921e19087e8 2024-11-23T16:16:03,658 DEBUG [StoreCloser-TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/A/72eae2668f414abd857521062e3d9477 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/A/72eae2668f414abd857521062e3d9477 2024-11-23T16:16:03,659 DEBUG [StoreCloser-TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/A/5e3be1e387b94f7c91c829c1071cadc1 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/A/5e3be1e387b94f7c91c829c1071cadc1 2024-11-23T16:16:03,659 DEBUG [StoreCloser-TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/A/b712b5711ed4432587b0c6c286d24466 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/A/b712b5711ed4432587b0c6c286d24466 2024-11-23T16:16:03,660 DEBUG [StoreCloser-TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/A/b75f3522fd4448da80b8cf120f65f080 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/A/b75f3522fd4448da80b8cf120f65f080 2024-11-23T16:16:03,661 DEBUG [StoreCloser-TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/A/5e891158ee0a4643a9bfb4c8def3d58c to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/A/5e891158ee0a4643a9bfb4c8def3d58c 2024-11-23T16:16:03,661 DEBUG [StoreCloser-TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/A/7d041460c2714d1ea1f05f3fdd91bafd to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/A/7d041460c2714d1ea1f05f3fdd91bafd 2024-11-23T16:16:03,662 DEBUG [StoreCloser-TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/A/f9fcba2216d74f96b2d6a5b3350db9c6 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/A/f9fcba2216d74f96b2d6a5b3350db9c6 2024-11-23T16:16:03,663 DEBUG [StoreCloser-TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/A/1726c374e7104d88bb522f416de35e4b to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/A/1726c374e7104d88bb522f416de35e4b 2024-11-23T16:16:03,663 DEBUG [StoreCloser-TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/A/e4e7bc9d208c47ddb66188e391225790 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/A/e4e7bc9d208c47ddb66188e391225790 2024-11-23T16:16:03,664 DEBUG [StoreCloser-TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/A/9ffada2e725445a3aec3b7ce1e781113 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/A/9ffada2e725445a3aec3b7ce1e781113 2024-11-23T16:16:03,665 DEBUG [StoreCloser-TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/A/0f3755b1e5c44a7a83a249f6e1b53a29 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/A/0f3755b1e5c44a7a83a249f6e1b53a29 2024-11-23T16:16:03,666 DEBUG [StoreCloser-TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/A/7e8a7c2c98464f0f9d746be30d7608dd to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/A/7e8a7c2c98464f0f9d746be30d7608dd 2024-11-23T16:16:03,667 DEBUG [StoreCloser-TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/A/388c96d9a56a4f998767a7a865f289dd to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/A/388c96d9a56a4f998767a7a865f289dd 2024-11-23T16:16:03,667 DEBUG [StoreCloser-TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/A/d5eca5bad3a547c8b6ecfd1a48b1f464 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/A/d5eca5bad3a547c8b6ecfd1a48b1f464 2024-11-23T16:16:03,668 DEBUG [StoreCloser-TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/A/fb6c56f5b0774eddbcddb5f874ccbd43 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/A/fb6c56f5b0774eddbcddb5f874ccbd43 2024-11-23T16:16:03,669 DEBUG [StoreCloser-TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/A/c4c89b2c3eb9412685e407c00147a596 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/A/c4c89b2c3eb9412685e407c00147a596 2024-11-23T16:16:03,670 DEBUG [StoreCloser-TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/B/c77e2cf5829b48a18e01b5ee7112880b, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/B/9795dabb27ea4c568cf1ef52fad9bf4f, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/B/df4916605cc1401c9bd809a39c46dec9, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/B/c07aec55d0ff47fdb88fae1987d71c5e, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/B/8a625a77afa2499d970a7d13f7c288ba, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/B/79f4376771074b9ea9180bc24137a064, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/B/eb993843f605415ca294ce328eb2875c, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/B/e6778eae86ab40e7bbb9e5e24c3abf08, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/B/d61c7db00f1f4f33b1eb6a71074e89dc, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/B/777861f36c1c48479784df932cec97ce, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/B/55ed716d7c3e4d0b9645c4df1d8b566f, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/B/984a6841474d4eeeaf864a5552cfa3d2, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/B/5a5972fa91e048f9b9e307e54020cb97, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/B/779aa8a1e548449ba38570f80380f917, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/B/c469a19218cb41a5b69e98708c12c425, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/B/cc5c598fd8124740ae4b3a70355f0561, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/B/16f97a0665064d5dab18689711840cb8, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/B/98d7df745342456997b51fd6577931d8, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/B/04670946fcbe4ae6adc273f7ca906ffe, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/B/11b0e8319b5842f39f82614d5d487b21, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/B/cb58dbba890d432580f79ee6b2c151be, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/B/9367c72ebcf149f0b22d94aaaa687317, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/B/d9927a7f610644f3a489fa4407a34f8f, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/B/3e2a12b146f840b9b0590000da83d051, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/B/ec230b48dfb345928edadfdc2fc3a778, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/B/afa1aae79c1041e28935011142718324, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/B/41c359a4030c4a93a374b36decace292, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/B/ed9715959c184b87b8e6361f91db3f45] to archive 2024-11-23T16:16:03,671 DEBUG [StoreCloser-TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-23T16:16:03,672 DEBUG [StoreCloser-TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/B/c77e2cf5829b48a18e01b5ee7112880b to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/B/c77e2cf5829b48a18e01b5ee7112880b 2024-11-23T16:16:03,673 DEBUG [StoreCloser-TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/B/9795dabb27ea4c568cf1ef52fad9bf4f to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/B/9795dabb27ea4c568cf1ef52fad9bf4f 2024-11-23T16:16:03,673 DEBUG [StoreCloser-TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/B/df4916605cc1401c9bd809a39c46dec9 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/B/df4916605cc1401c9bd809a39c46dec9 2024-11-23T16:16:03,674 DEBUG [StoreCloser-TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/B/c07aec55d0ff47fdb88fae1987d71c5e to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/B/c07aec55d0ff47fdb88fae1987d71c5e 2024-11-23T16:16:03,675 DEBUG [StoreCloser-TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/B/8a625a77afa2499d970a7d13f7c288ba to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/B/8a625a77afa2499d970a7d13f7c288ba 2024-11-23T16:16:03,675 DEBUG [StoreCloser-TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/B/79f4376771074b9ea9180bc24137a064 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/B/79f4376771074b9ea9180bc24137a064 2024-11-23T16:16:03,676 DEBUG [StoreCloser-TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/B/eb993843f605415ca294ce328eb2875c to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/B/eb993843f605415ca294ce328eb2875c 2024-11-23T16:16:03,677 DEBUG [StoreCloser-TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/B/e6778eae86ab40e7bbb9e5e24c3abf08 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/B/e6778eae86ab40e7bbb9e5e24c3abf08 2024-11-23T16:16:03,678 DEBUG [StoreCloser-TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/B/d61c7db00f1f4f33b1eb6a71074e89dc to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/B/d61c7db00f1f4f33b1eb6a71074e89dc 2024-11-23T16:16:03,678 DEBUG [StoreCloser-TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/B/777861f36c1c48479784df932cec97ce to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/B/777861f36c1c48479784df932cec97ce 2024-11-23T16:16:03,679 DEBUG [StoreCloser-TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/B/55ed716d7c3e4d0b9645c4df1d8b566f to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/B/55ed716d7c3e4d0b9645c4df1d8b566f 2024-11-23T16:16:03,680 DEBUG [StoreCloser-TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/B/984a6841474d4eeeaf864a5552cfa3d2 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/B/984a6841474d4eeeaf864a5552cfa3d2 2024-11-23T16:16:03,680 DEBUG [StoreCloser-TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/B/5a5972fa91e048f9b9e307e54020cb97 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/B/5a5972fa91e048f9b9e307e54020cb97 2024-11-23T16:16:03,681 DEBUG [StoreCloser-TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/B/779aa8a1e548449ba38570f80380f917 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/B/779aa8a1e548449ba38570f80380f917 2024-11-23T16:16:03,682 DEBUG [StoreCloser-TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/B/c469a19218cb41a5b69e98708c12c425 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/B/c469a19218cb41a5b69e98708c12c425 2024-11-23T16:16:03,683 DEBUG [StoreCloser-TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/B/cc5c598fd8124740ae4b3a70355f0561 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/B/cc5c598fd8124740ae4b3a70355f0561 2024-11-23T16:16:03,683 DEBUG [StoreCloser-TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/B/16f97a0665064d5dab18689711840cb8 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/B/16f97a0665064d5dab18689711840cb8 2024-11-23T16:16:03,684 DEBUG [StoreCloser-TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/B/98d7df745342456997b51fd6577931d8 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/B/98d7df745342456997b51fd6577931d8 2024-11-23T16:16:03,685 DEBUG [StoreCloser-TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/B/04670946fcbe4ae6adc273f7ca906ffe to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/B/04670946fcbe4ae6adc273f7ca906ffe 2024-11-23T16:16:03,686 DEBUG [StoreCloser-TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/B/11b0e8319b5842f39f82614d5d487b21 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/B/11b0e8319b5842f39f82614d5d487b21 2024-11-23T16:16:03,686 DEBUG [StoreCloser-TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/B/cb58dbba890d432580f79ee6b2c151be to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/B/cb58dbba890d432580f79ee6b2c151be 2024-11-23T16:16:03,687 DEBUG [StoreCloser-TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/B/9367c72ebcf149f0b22d94aaaa687317 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/B/9367c72ebcf149f0b22d94aaaa687317 2024-11-23T16:16:03,688 DEBUG [StoreCloser-TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/B/d9927a7f610644f3a489fa4407a34f8f to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/B/d9927a7f610644f3a489fa4407a34f8f 2024-11-23T16:16:03,688 DEBUG [StoreCloser-TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/B/3e2a12b146f840b9b0590000da83d051 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/B/3e2a12b146f840b9b0590000da83d051 2024-11-23T16:16:03,689 DEBUG [StoreCloser-TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/B/ec230b48dfb345928edadfdc2fc3a778 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/B/ec230b48dfb345928edadfdc2fc3a778 2024-11-23T16:16:03,690 DEBUG [StoreCloser-TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/B/afa1aae79c1041e28935011142718324 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/B/afa1aae79c1041e28935011142718324 2024-11-23T16:16:03,691 DEBUG [StoreCloser-TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/B/41c359a4030c4a93a374b36decace292 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/B/41c359a4030c4a93a374b36decace292 2024-11-23T16:16:03,691 DEBUG [StoreCloser-TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/B/ed9715959c184b87b8e6361f91db3f45 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/B/ed9715959c184b87b8e6361f91db3f45 2024-11-23T16:16:03,692 DEBUG [StoreCloser-TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/C/b50cebd05b6a43169e0d7a0e730aa3b9, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/C/a50a1449830142ef80ec28c72a09b387, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/C/2f81d9d5266d4015915d2cbccb677729, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/C/7ca85a5b077544a2aea19ab716007e42, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/C/457649d6dd8b4269824f8f975d3c4c42, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/C/c4d416dc87864934a5bda95181df5631, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/C/9f3f57eff44c48adbd1caffd0ce3b246, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/C/77ae4cd18d95457a800dce425811621a, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/C/222fcf65d0e34f709f6bc09e0d98cb8c, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/C/4b1dbabea1634cd7b198ae442084faa2, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/C/3d648d05e92148d3a0e663b21f08e739, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/C/5a0ed7d8fd344976b7cc8f250d603503, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/C/cabd0470aaaa494b8278d295b608ab28, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/C/b71d529847614da0bc5d647651606101, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/C/7a7b449f55c84b91a8a38c77e259b635, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/C/a44d44c2a1db4af98aeede55cb9899c5, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/C/f451d23b7f304d1cac2cb312a38829d0, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/C/52f70de1ad22450da38441bcfec9ea4e, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/C/25280f7dea6c4d6eb15a2b71955f5c20, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/C/661d7c4f70b64c00a8c8580e7bd0c276, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/C/6f359f53a70943088ebd5269b37fae5b, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/C/724b560fe3894a9aa2ec63a6e4f1873e, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/C/dc979853a15d4ca1b8a617e6db03db34, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/C/00618d75c3a94f639cbdb10f360e8f5b, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/C/8c38b95c143f41d3b49faa2e97b8b89c, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/C/6f812fd0d0bb43d495ffb1049646cfa3, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/C/aa76e434783242f3a6d8d24020a71920] to archive 2024-11-23T16:16:03,693 DEBUG [StoreCloser-TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-23T16:16:03,694 DEBUG [StoreCloser-TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/C/b50cebd05b6a43169e0d7a0e730aa3b9 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/C/b50cebd05b6a43169e0d7a0e730aa3b9 2024-11-23T16:16:03,695 DEBUG [StoreCloser-TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/C/a50a1449830142ef80ec28c72a09b387 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/C/a50a1449830142ef80ec28c72a09b387 2024-11-23T16:16:03,696 DEBUG [StoreCloser-TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/C/2f81d9d5266d4015915d2cbccb677729 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/C/2f81d9d5266d4015915d2cbccb677729 2024-11-23T16:16:03,696 DEBUG [StoreCloser-TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/C/7ca85a5b077544a2aea19ab716007e42 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/C/7ca85a5b077544a2aea19ab716007e42 2024-11-23T16:16:03,697 DEBUG [StoreCloser-TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/C/457649d6dd8b4269824f8f975d3c4c42 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/C/457649d6dd8b4269824f8f975d3c4c42 2024-11-23T16:16:03,698 DEBUG [StoreCloser-TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/C/c4d416dc87864934a5bda95181df5631 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/C/c4d416dc87864934a5bda95181df5631 2024-11-23T16:16:03,698 DEBUG [StoreCloser-TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/C/9f3f57eff44c48adbd1caffd0ce3b246 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/C/9f3f57eff44c48adbd1caffd0ce3b246 2024-11-23T16:16:03,699 DEBUG [StoreCloser-TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/C/77ae4cd18d95457a800dce425811621a to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/C/77ae4cd18d95457a800dce425811621a 2024-11-23T16:16:03,700 DEBUG [StoreCloser-TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/C/222fcf65d0e34f709f6bc09e0d98cb8c to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/C/222fcf65d0e34f709f6bc09e0d98cb8c 2024-11-23T16:16:03,701 DEBUG [StoreCloser-TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/C/4b1dbabea1634cd7b198ae442084faa2 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/C/4b1dbabea1634cd7b198ae442084faa2 2024-11-23T16:16:03,701 DEBUG [StoreCloser-TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/C/3d648d05e92148d3a0e663b21f08e739 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/C/3d648d05e92148d3a0e663b21f08e739 2024-11-23T16:16:03,702 DEBUG [StoreCloser-TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/C/5a0ed7d8fd344976b7cc8f250d603503 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/C/5a0ed7d8fd344976b7cc8f250d603503 2024-11-23T16:16:03,703 DEBUG [StoreCloser-TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/C/cabd0470aaaa494b8278d295b608ab28 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/C/cabd0470aaaa494b8278d295b608ab28 2024-11-23T16:16:03,703 DEBUG [StoreCloser-TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/C/b71d529847614da0bc5d647651606101 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/C/b71d529847614da0bc5d647651606101 2024-11-23T16:16:03,704 DEBUG [StoreCloser-TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/C/7a7b449f55c84b91a8a38c77e259b635 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/C/7a7b449f55c84b91a8a38c77e259b635 2024-11-23T16:16:03,705 DEBUG [StoreCloser-TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/C/a44d44c2a1db4af98aeede55cb9899c5 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/C/a44d44c2a1db4af98aeede55cb9899c5 2024-11-23T16:16:03,706 DEBUG [StoreCloser-TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/C/f451d23b7f304d1cac2cb312a38829d0 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/C/f451d23b7f304d1cac2cb312a38829d0 2024-11-23T16:16:03,707 DEBUG [StoreCloser-TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/C/52f70de1ad22450da38441bcfec9ea4e to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/C/52f70de1ad22450da38441bcfec9ea4e 2024-11-23T16:16:03,707 DEBUG [StoreCloser-TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/C/25280f7dea6c4d6eb15a2b71955f5c20 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/C/25280f7dea6c4d6eb15a2b71955f5c20 2024-11-23T16:16:03,708 DEBUG [StoreCloser-TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/C/661d7c4f70b64c00a8c8580e7bd0c276 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/C/661d7c4f70b64c00a8c8580e7bd0c276 2024-11-23T16:16:03,709 DEBUG [StoreCloser-TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/C/6f359f53a70943088ebd5269b37fae5b to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/C/6f359f53a70943088ebd5269b37fae5b 2024-11-23T16:16:03,710 DEBUG [StoreCloser-TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/C/724b560fe3894a9aa2ec63a6e4f1873e to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/C/724b560fe3894a9aa2ec63a6e4f1873e 2024-11-23T16:16:03,711 DEBUG [StoreCloser-TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/C/dc979853a15d4ca1b8a617e6db03db34 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/C/dc979853a15d4ca1b8a617e6db03db34 2024-11-23T16:16:03,711 DEBUG [StoreCloser-TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/C/00618d75c3a94f639cbdb10f360e8f5b to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/C/00618d75c3a94f639cbdb10f360e8f5b 2024-11-23T16:16:03,712 DEBUG [StoreCloser-TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/C/8c38b95c143f41d3b49faa2e97b8b89c to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/C/8c38b95c143f41d3b49faa2e97b8b89c 2024-11-23T16:16:03,713 DEBUG [StoreCloser-TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/C/6f812fd0d0bb43d495ffb1049646cfa3 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/C/6f812fd0d0bb43d495ffb1049646cfa3 2024-11-23T16:16:03,713 DEBUG [StoreCloser-TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/C/aa76e434783242f3a6d8d24020a71920 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/C/aa76e434783242f3a6d8d24020a71920 2024-11-23T16:16:03,717 DEBUG [RS_CLOSE_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/recovered.edits/462.seqid, newMaxSeqId=462, maxSeqId=1 2024-11-23T16:16:03,718 INFO [RS_CLOSE_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4. 2024-11-23T16:16:03,718 DEBUG [RS_CLOSE_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] regionserver.HRegion(1676): Region close journal for 63f43c63344ece03d182c42497a352e4: Waiting for close lock at 1732378561499Running coprocessor pre-close hooks at 1732378561499Disabling compacts and flushes for region at 1732378561499Disabling writes for close at 1732378562410 (+911 ms)Obtaining lock to block concurrent updates at 1732378562410Preparing flush snapshotting stores in 63f43c63344ece03d182c42497a352e4 at 1732378562410Finished memstore snapshotting TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4., syncing WAL and waiting on mvcc, flushsize=dataSize=96180, getHeapSize=266976, getOffHeapSize=0, getCellsCount=2100 at 1732378562410Flushing stores of TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4. at 1732378562410Flushing 63f43c63344ece03d182c42497a352e4/A: creating writer at 1732378562411 (+1 ms)Flushing 63f43c63344ece03d182c42497a352e4/A: appending metadata at 1732378562413 (+2 ms)Flushing 63f43c63344ece03d182c42497a352e4/A: closing flushed file at 1732378562413Flushing 63f43c63344ece03d182c42497a352e4/B: creating writer at 1732378562819 (+406 ms)Flushing 63f43c63344ece03d182c42497a352e4/B: appending metadata at 1732378562822 (+3 ms)Flushing 63f43c63344ece03d182c42497a352e4/B: closing flushed file at 1732378562822Flushing 63f43c63344ece03d182c42497a352e4/C: creating writer at 1732378563228 (+406 ms)Flushing 63f43c63344ece03d182c42497a352e4/C: appending metadata at 1732378563230 (+2 ms)Flushing 63f43c63344ece03d182c42497a352e4/C: closing flushed file at 1732378563230Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@28e90037: reopening flushed file at 1732378563637 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@52d6d066: reopening flushed file at 1732378563640 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2d94d626: reopening flushed file at 1732378563643 (+3 ms)Finished flush of dataSize ~93.93 KB/96180, heapSize ~260.72 KB/266976, currentSize=0 B/0 for 63f43c63344ece03d182c42497a352e4 in 1237ms, sequenceid=459, compaction requested=true at 1732378563647 (+4 ms)Writing region close event to WAL at 1732378563715 (+68 ms)Running coprocessor post-close hooks at 1732378563718 (+3 ms)Closed at 1732378563718 2024-11-23T16:16:03,720 INFO [RS_CLOSE_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] handler.UnassignRegionHandler(157): Closed 63f43c63344ece03d182c42497a352e4 2024-11-23T16:16:03,720 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=129 updating hbase:meta row=63f43c63344ece03d182c42497a352e4, regionState=CLOSED 2024-11-23T16:16:03,721 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=130, ppid=129, state=RUNNABLE, hasLock=false; CloseRegionProcedure 63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 because future has completed 2024-11-23T16:16:03,723 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=130, resume processing ppid=129 2024-11-23T16:16:03,723 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=130, ppid=129, state=SUCCESS, hasLock=false; CloseRegionProcedure 63f43c63344ece03d182c42497a352e4, server=1a7c5470b472,44715,1732378420375 in 2.3750 sec 2024-11-23T16:16:03,724 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=129, resume processing ppid=128 2024-11-23T16:16:03,724 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=129, ppid=128, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=63f43c63344ece03d182c42497a352e4, UNASSIGN in 2.3790 sec 2024-11-23T16:16:03,725 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=128, resume processing ppid=127 2024-11-23T16:16:03,726 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=128, ppid=127, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 2.3810 sec 2024-11-23T16:16:03,726 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732378563726"}]},"ts":"1732378563726"} 2024-11-23T16:16:03,727 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-23T16:16:03,727 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-23T16:16:03,729 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=127, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 2.3890 sec 2024-11-23T16:16:05,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-23T16:16:05,508 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-23T16:16:05,508 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-23T16:16:05,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] procedure2.ProcedureExecutor(1139): Stored pid=131, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-23T16:16:05,510 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=131, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-23T16:16:05,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-23T16:16:05,510 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=131, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-23T16:16:05,513 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4 2024-11-23T16:16:05,515 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/A, FileablePath, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/B, FileablePath, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/C, FileablePath, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/recovered.edits] 2024-11-23T16:16:05,517 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/A/072612d169b74c6a884de1fe09d96cca to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/A/072612d169b74c6a884de1fe09d96cca 2024-11-23T16:16:05,518 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/A/27ebffaf4ce9489b88ea9bf4ef2669d9 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/A/27ebffaf4ce9489b88ea9bf4ef2669d9 2024-11-23T16:16:05,519 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/A/c05147b5dcea44c290714de5d7676bf0 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/A/c05147b5dcea44c290714de5d7676bf0 2024-11-23T16:16:05,520 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/A/dfd7a28e34614d8ab4c9f15f4d03a6d3 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/A/dfd7a28e34614d8ab4c9f15f4d03a6d3 2024-11-23T16:16:05,521 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/B/c11ef502b4d943b19f4bc54c0b9014cb to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/B/c11ef502b4d943b19f4bc54c0b9014cb 2024-11-23T16:16:05,522 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/B/ce975a5dd97943a783adf2b915e28562 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/B/ce975a5dd97943a783adf2b915e28562 2024-11-23T16:16:05,523 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/B/f2eff5e552974887aceeda069398a33d to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/B/f2eff5e552974887aceeda069398a33d 2024-11-23T16:16:05,524 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/B/f4359489557b4d65b3f287c81b23589a to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/B/f4359489557b4d65b3f287c81b23589a 2024-11-23T16:16:05,526 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/C/35832a69ef2540e7b7b0a9a9075f92e9 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/C/35832a69ef2540e7b7b0a9a9075f92e9 2024-11-23T16:16:05,526 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/C/5ea77280771a4ae69d8821d8e76852a4 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/C/5ea77280771a4ae69d8821d8e76852a4 2024-11-23T16:16:05,527 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/C/71386003914b42d683b04091a7efa642 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/C/71386003914b42d683b04091a7efa642 2024-11-23T16:16:05,528 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/C/c78f575db88345ccb9112bf85711ca14 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/C/c78f575db88345ccb9112bf85711ca14 2024-11-23T16:16:05,530 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/recovered.edits/462.seqid to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4/recovered.edits/462.seqid 2024-11-23T16:16:05,531 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/63f43c63344ece03d182c42497a352e4 2024-11-23T16:16:05,531 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-23T16:16:05,532 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=131, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-23T16:16:05,534 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-23T16:16:05,536 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-23T16:16:05,537 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=131, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-23T16:16:05,537 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-23T16:16:05,537 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732378565537"}]},"ts":"9223372036854775807"} 2024-11-23T16:16:05,538 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-23T16:16:05,538 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 63f43c63344ece03d182c42497a352e4, NAME => 'TestAcidGuarantees,,1732378539027.63f43c63344ece03d182c42497a352e4.', STARTKEY => '', ENDKEY => ''}] 2024-11-23T16:16:05,538 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-23T16:16:05,539 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732378565538"}]},"ts":"9223372036854775807"} 2024-11-23T16:16:05,540 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-23T16:16:05,540 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=131, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-23T16:16:05,541 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=131, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 32 msec 2024-11-23T16:16:05,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-23T16:16:05,618 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-23T16:16:05,618 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-23T16:16:05,627 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testScanAtomicity Thread=250 (was 251), OpenFileDescriptor=449 (was 459), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=395 (was 386) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=6735 (was 6810) 2024-11-23T16:16:05,635 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobGetAtomicity Thread=250, OpenFileDescriptor=449, MaxFileDescriptor=1048576, SystemLoadAverage=395, ProcessCount=11, AvailableMemoryMB=6734 2024-11-23T16:16:05,636 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-23T16:16:05,637 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-23T16:16:05,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] procedure2.ProcedureExecutor(1139): Stored pid=132, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-23T16:16:05,638 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=132, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-23T16:16:05,639 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 132 2024-11-23T16:16:05,639 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:05,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=132 2024-11-23T16:16:05,639 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=132, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-23T16:16:05,645 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742336_1512 (size=1061) 2024-11-23T16:16:05,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=132 2024-11-23T16:16:05,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=132 2024-11-23T16:16:06,047 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 1930901e64343eeb48a7930647197c36, NAME => 'TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca 2024-11-23T16:16:06,052 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742337_1513 (size=53) 2024-11-23T16:16:06,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=132 2024-11-23T16:16:06,453 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T16:16:06,453 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 1930901e64343eeb48a7930647197c36, disabling compactions & flushes 2024-11-23T16:16:06,453 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36. 2024-11-23T16:16:06,453 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36. 2024-11-23T16:16:06,453 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36. after waiting 0 ms 2024-11-23T16:16:06,453 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36. 2024-11-23T16:16:06,453 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36. 2024-11-23T16:16:06,453 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 1930901e64343eeb48a7930647197c36: Waiting for close lock at 1732378566453Disabling compacts and flushes for region at 1732378566453Disabling writes for close at 1732378566453Writing region close event to WAL at 1732378566453Closed at 1732378566453 2024-11-23T16:16:06,454 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=132, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-23T16:16:06,454 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732378566454"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732378566454"}]},"ts":"1732378566454"} 2024-11-23T16:16:06,456 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-23T16:16:06,456 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=132, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-23T16:16:06,456 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732378566456"}]},"ts":"1732378566456"} 2024-11-23T16:16:06,457 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-23T16:16:06,457 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=133, ppid=132, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1930901e64343eeb48a7930647197c36, ASSIGN}] 2024-11-23T16:16:06,458 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=133, ppid=132, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1930901e64343eeb48a7930647197c36, ASSIGN 2024-11-23T16:16:06,459 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=133, ppid=132, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=1930901e64343eeb48a7930647197c36, ASSIGN; state=OFFLINE, location=1a7c5470b472,44715,1732378420375; forceNewPlan=false, retain=false 2024-11-23T16:16:06,609 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=133 updating hbase:meta row=1930901e64343eeb48a7930647197c36, regionState=OPENING, regionLocation=1a7c5470b472,44715,1732378420375 2024-11-23T16:16:06,611 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=133, ppid=132, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=1930901e64343eeb48a7930647197c36, ASSIGN because future has completed 2024-11-23T16:16:06,611 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=134, ppid=133, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375}] 2024-11-23T16:16:06,766 INFO [RS_OPEN_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_OPEN_REGION, pid=134}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36. 2024-11-23T16:16:06,766 DEBUG [RS_OPEN_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_OPEN_REGION, pid=134}] regionserver.HRegion(7752): Opening region: {ENCODED => 1930901e64343eeb48a7930647197c36, NAME => 'TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36.', STARTKEY => '', ENDKEY => ''} 2024-11-23T16:16:06,766 DEBUG [RS_OPEN_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_OPEN_REGION, pid=134}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 1930901e64343eeb48a7930647197c36 2024-11-23T16:16:06,766 DEBUG [RS_OPEN_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_OPEN_REGION, pid=134}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T16:16:06,766 DEBUG [RS_OPEN_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_OPEN_REGION, pid=134}] regionserver.HRegion(7794): checking encryption for 1930901e64343eeb48a7930647197c36 2024-11-23T16:16:06,766 DEBUG [RS_OPEN_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_OPEN_REGION, pid=134}] regionserver.HRegion(7797): checking classloading for 1930901e64343eeb48a7930647197c36 2024-11-23T16:16:06,767 INFO [StoreOpener-1930901e64343eeb48a7930647197c36-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 1930901e64343eeb48a7930647197c36 2024-11-23T16:16:06,768 INFO [StoreOpener-1930901e64343eeb48a7930647197c36-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-23T16:16:06,769 INFO [StoreOpener-1930901e64343eeb48a7930647197c36-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1930901e64343eeb48a7930647197c36 columnFamilyName A 2024-11-23T16:16:06,769 DEBUG [StoreOpener-1930901e64343eeb48a7930647197c36-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:06,769 INFO [StoreOpener-1930901e64343eeb48a7930647197c36-1 {}] regionserver.HStore(327): Store=1930901e64343eeb48a7930647197c36/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T16:16:06,769 INFO [StoreOpener-1930901e64343eeb48a7930647197c36-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 1930901e64343eeb48a7930647197c36 2024-11-23T16:16:06,770 INFO [StoreOpener-1930901e64343eeb48a7930647197c36-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-23T16:16:06,770 INFO [StoreOpener-1930901e64343eeb48a7930647197c36-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1930901e64343eeb48a7930647197c36 columnFamilyName B 2024-11-23T16:16:06,770 DEBUG [StoreOpener-1930901e64343eeb48a7930647197c36-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:06,770 INFO [StoreOpener-1930901e64343eeb48a7930647197c36-1 {}] regionserver.HStore(327): Store=1930901e64343eeb48a7930647197c36/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T16:16:06,770 INFO [StoreOpener-1930901e64343eeb48a7930647197c36-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 1930901e64343eeb48a7930647197c36 2024-11-23T16:16:06,771 INFO [StoreOpener-1930901e64343eeb48a7930647197c36-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-23T16:16:06,771 INFO [StoreOpener-1930901e64343eeb48a7930647197c36-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1930901e64343eeb48a7930647197c36 columnFamilyName C 2024-11-23T16:16:06,771 DEBUG [StoreOpener-1930901e64343eeb48a7930647197c36-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:06,771 INFO [StoreOpener-1930901e64343eeb48a7930647197c36-1 {}] regionserver.HStore(327): Store=1930901e64343eeb48a7930647197c36/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T16:16:06,771 INFO [RS_OPEN_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_OPEN_REGION, pid=134}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36. 2024-11-23T16:16:06,772 DEBUG [RS_OPEN_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_OPEN_REGION, pid=134}] regionserver.HRegion(1038): replaying wal for 1930901e64343eeb48a7930647197c36 2024-11-23T16:16:06,772 DEBUG [RS_OPEN_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_OPEN_REGION, pid=134}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36 2024-11-23T16:16:06,772 DEBUG [RS_OPEN_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_OPEN_REGION, pid=134}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36 2024-11-23T16:16:06,773 DEBUG [RS_OPEN_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_OPEN_REGION, pid=134}] regionserver.HRegion(1048): stopping wal replay for 1930901e64343eeb48a7930647197c36 2024-11-23T16:16:06,773 DEBUG [RS_OPEN_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_OPEN_REGION, pid=134}] regionserver.HRegion(1060): Cleaning up temporary data for 1930901e64343eeb48a7930647197c36 2024-11-23T16:16:06,773 DEBUG [RS_OPEN_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_OPEN_REGION, pid=134}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-23T16:16:06,774 DEBUG [RS_OPEN_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_OPEN_REGION, pid=134}] regionserver.HRegion(1093): writing seq id for 1930901e64343eeb48a7930647197c36 2024-11-23T16:16:06,775 DEBUG [RS_OPEN_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_OPEN_REGION, pid=134}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-23T16:16:06,776 INFO [RS_OPEN_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_OPEN_REGION, pid=134}] regionserver.HRegion(1114): Opened 1930901e64343eeb48a7930647197c36; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66164581, jitterRate=-0.014070913195610046}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-23T16:16:06,776 DEBUG [RS_OPEN_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_OPEN_REGION, pid=134}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1930901e64343eeb48a7930647197c36 2024-11-23T16:16:06,776 DEBUG [RS_OPEN_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_OPEN_REGION, pid=134}] regionserver.HRegion(1006): Region open journal for 1930901e64343eeb48a7930647197c36: Running coprocessor pre-open hook at 1732378566766Writing region info on filesystem at 1732378566766Initializing all the Stores at 1732378566767 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732378566767Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732378566767Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732378566767Cleaning up temporary data from old regions at 1732378566773 (+6 ms)Running coprocessor post-open hooks at 1732378566776 (+3 ms)Region opened successfully at 1732378566776 2024-11-23T16:16:06,777 INFO [RS_OPEN_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_OPEN_REGION, pid=134}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., pid=134, masterSystemTime=1732378566762 2024-11-23T16:16:06,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=132 2024-11-23T16:16:06,778 DEBUG [RS_OPEN_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_OPEN_REGION, pid=134}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36. 2024-11-23T16:16:06,778 INFO [RS_OPEN_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_OPEN_REGION, pid=134}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36. 2024-11-23T16:16:06,779 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=133 updating hbase:meta row=1930901e64343eeb48a7930647197c36, regionState=OPEN, openSeqNum=2, regionLocation=1a7c5470b472,44715,1732378420375 2024-11-23T16:16:06,780 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=134, ppid=133, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 because future has completed 2024-11-23T16:16:06,782 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=134, resume processing ppid=133 2024-11-23T16:16:06,782 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=134, ppid=133, state=SUCCESS, hasLock=false; OpenRegionProcedure 1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 in 169 msec 2024-11-23T16:16:06,783 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=133, resume processing ppid=132 2024-11-23T16:16:06,783 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=133, ppid=132, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1930901e64343eeb48a7930647197c36, ASSIGN in 325 msec 2024-11-23T16:16:06,783 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=132, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-23T16:16:06,784 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732378566784"}]},"ts":"1732378566784"} 2024-11-23T16:16:06,785 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-23T16:16:06,785 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=132, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-23T16:16:06,786 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=132, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1480 sec 2024-11-23T16:16:07,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=132 2024-11-23T16:16:07,788 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-23T16:16:07,789 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@a04de68, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T16:16:07,789 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1a7c5470b472,45799,-1 for getting cluster id 2024-11-23T16:16:07,789 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-23T16:16:07,790 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '34e0592b-548a-4d73-be6a-df1bafadfd92' 2024-11-23T16:16:07,790 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-23T16:16:07,790 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "34e0592b-548a-4d73-be6a-df1bafadfd92" 2024-11-23T16:16:07,791 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@50ab5178, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T16:16:07,791 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1a7c5470b472,45799,-1] 2024-11-23T16:16:07,791 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-23T16:16:07,791 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T16:16:07,792 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36256, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-23T16:16:07,792 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2ecba0f8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T16:16:07,792 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T16:16:07,793 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1a7c5470b472,44715,1732378420375, seqNum=-1] 2024-11-23T16:16:07,793 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T16:16:07,794 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53754, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T16:16:07,795 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-23T16:16:07,795 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncConnectionImpl(321): The fetched master address is 1a7c5470b472,45799,1732378419617 2024-11-23T16:16:07,795 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@26f081e7 2024-11-23T16:16:07,795 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-23T16:16:07,796 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36258, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-23T16:16:07,797 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-23T16:16:07,797 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-23T16:16:07,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] procedure2.ProcedureExecutor(1139): Stored pid=135, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-23T16:16:07,807 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742338_1514 (size=1097) 2024-11-23T16:16:08,208 DEBUG [PEWorker-1 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-11-23T16:16:08,208 INFO [PEWorker-1 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-11-23T16:16:08,211 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=136, ppid=135, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-23T16:16:08,212 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=137, ppid=136, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1930901e64343eeb48a7930647197c36, REOPEN/MOVE}] 2024-11-23T16:16:08,213 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=137, ppid=136, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1930901e64343eeb48a7930647197c36, REOPEN/MOVE 2024-11-23T16:16:08,214 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=137 updating hbase:meta row=1930901e64343eeb48a7930647197c36, regionState=CLOSING, regionLocation=1a7c5470b472,44715,1732378420375 2024-11-23T16:16:08,215 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=137, ppid=136, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=1930901e64343eeb48a7930647197c36, REOPEN/MOVE because future has completed 2024-11-23T16:16:08,215 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-23T16:16:08,215 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=138, ppid=137, state=RUNNABLE, hasLock=false; CloseRegionProcedure 1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375}] 2024-11-23T16:16:08,367 INFO [RS_CLOSE_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] handler.UnassignRegionHandler(122): Close 1930901e64343eeb48a7930647197c36 2024-11-23T16:16:08,367 DEBUG [RS_CLOSE_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-23T16:16:08,367 DEBUG [RS_CLOSE_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegion(1722): Closing 1930901e64343eeb48a7930647197c36, disabling compactions & flushes 2024-11-23T16:16:08,367 INFO [RS_CLOSE_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36. 2024-11-23T16:16:08,367 DEBUG [RS_CLOSE_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36. 2024-11-23T16:16:08,367 DEBUG [RS_CLOSE_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36. after waiting 0 ms 2024-11-23T16:16:08,367 DEBUG [RS_CLOSE_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36. 2024-11-23T16:16:08,370 DEBUG [RS_CLOSE_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-23T16:16:08,371 INFO [RS_CLOSE_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36. 2024-11-23T16:16:08,371 DEBUG [RS_CLOSE_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegion(1676): Region close journal for 1930901e64343eeb48a7930647197c36: Waiting for close lock at 1732378568367Running coprocessor pre-close hooks at 1732378568367Disabling compacts and flushes for region at 1732378568367Disabling writes for close at 1732378568367Writing region close event to WAL at 1732378568367Running coprocessor post-close hooks at 1732378568370 (+3 ms)Closed at 1732378568370 2024-11-23T16:16:08,371 WARN [RS_CLOSE_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegionServer(3299): Not adding moved region record: 1930901e64343eeb48a7930647197c36 to self. 2024-11-23T16:16:08,372 INFO [RS_CLOSE_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] handler.UnassignRegionHandler(157): Closed 1930901e64343eeb48a7930647197c36 2024-11-23T16:16:08,372 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=137 updating hbase:meta row=1930901e64343eeb48a7930647197c36, regionState=CLOSED 2024-11-23T16:16:08,373 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=138, ppid=137, state=RUNNABLE, hasLock=false; CloseRegionProcedure 1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 because future has completed 2024-11-23T16:16:08,375 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=138, resume processing ppid=137 2024-11-23T16:16:08,375 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=138, ppid=137, state=SUCCESS, hasLock=false; CloseRegionProcedure 1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 in 159 msec 2024-11-23T16:16:08,376 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=137, ppid=136, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=1930901e64343eeb48a7930647197c36, REOPEN/MOVE; state=CLOSED, location=1a7c5470b472,44715,1732378420375; forceNewPlan=false, retain=true 2024-11-23T16:16:08,526 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=137 updating hbase:meta row=1930901e64343eeb48a7930647197c36, regionState=OPENING, regionLocation=1a7c5470b472,44715,1732378420375 2024-11-23T16:16:08,528 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=137, ppid=136, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=1930901e64343eeb48a7930647197c36, REOPEN/MOVE because future has completed 2024-11-23T16:16:08,528 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=139, ppid=137, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375}] 2024-11-23T16:16:08,631 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-23T16:16:08,682 INFO [RS_OPEN_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_OPEN_REGION, pid=139}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36. 2024-11-23T16:16:08,682 DEBUG [RS_OPEN_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_OPEN_REGION, pid=139}] regionserver.HRegion(7752): Opening region: {ENCODED => 1930901e64343eeb48a7930647197c36, NAME => 'TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36.', STARTKEY => '', ENDKEY => ''} 2024-11-23T16:16:08,682 DEBUG [RS_OPEN_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_OPEN_REGION, pid=139}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 1930901e64343eeb48a7930647197c36 2024-11-23T16:16:08,682 DEBUG [RS_OPEN_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_OPEN_REGION, pid=139}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T16:16:08,683 DEBUG [RS_OPEN_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_OPEN_REGION, pid=139}] regionserver.HRegion(7794): checking encryption for 1930901e64343eeb48a7930647197c36 2024-11-23T16:16:08,683 DEBUG [RS_OPEN_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_OPEN_REGION, pid=139}] regionserver.HRegion(7797): checking classloading for 1930901e64343eeb48a7930647197c36 2024-11-23T16:16:08,684 INFO [StoreOpener-1930901e64343eeb48a7930647197c36-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 1930901e64343eeb48a7930647197c36 2024-11-23T16:16:08,684 INFO [StoreOpener-1930901e64343eeb48a7930647197c36-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-23T16:16:08,684 INFO [StoreOpener-1930901e64343eeb48a7930647197c36-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1930901e64343eeb48a7930647197c36 columnFamilyName A 2024-11-23T16:16:08,685 DEBUG [StoreOpener-1930901e64343eeb48a7930647197c36-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:08,685 INFO [StoreOpener-1930901e64343eeb48a7930647197c36-1 {}] regionserver.HStore(327): Store=1930901e64343eeb48a7930647197c36/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T16:16:08,686 INFO [StoreOpener-1930901e64343eeb48a7930647197c36-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 1930901e64343eeb48a7930647197c36 2024-11-23T16:16:08,686 INFO [StoreOpener-1930901e64343eeb48a7930647197c36-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-23T16:16:08,686 INFO [StoreOpener-1930901e64343eeb48a7930647197c36-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1930901e64343eeb48a7930647197c36 columnFamilyName B 2024-11-23T16:16:08,686 DEBUG [StoreOpener-1930901e64343eeb48a7930647197c36-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:08,687 INFO [StoreOpener-1930901e64343eeb48a7930647197c36-1 {}] regionserver.HStore(327): Store=1930901e64343eeb48a7930647197c36/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T16:16:08,687 INFO [StoreOpener-1930901e64343eeb48a7930647197c36-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 1930901e64343eeb48a7930647197c36 2024-11-23T16:16:08,687 INFO [StoreOpener-1930901e64343eeb48a7930647197c36-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-23T16:16:08,687 INFO [StoreOpener-1930901e64343eeb48a7930647197c36-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1930901e64343eeb48a7930647197c36 columnFamilyName C 2024-11-23T16:16:08,687 DEBUG [StoreOpener-1930901e64343eeb48a7930647197c36-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:08,688 INFO [StoreOpener-1930901e64343eeb48a7930647197c36-1 {}] regionserver.HStore(327): Store=1930901e64343eeb48a7930647197c36/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T16:16:08,688 INFO [RS_OPEN_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_OPEN_REGION, pid=139}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36. 2024-11-23T16:16:08,688 DEBUG [RS_OPEN_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_OPEN_REGION, pid=139}] regionserver.HRegion(1038): replaying wal for 1930901e64343eeb48a7930647197c36 2024-11-23T16:16:08,688 DEBUG [RS_OPEN_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_OPEN_REGION, pid=139}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36 2024-11-23T16:16:08,689 DEBUG [RS_OPEN_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_OPEN_REGION, pid=139}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36 2024-11-23T16:16:08,690 DEBUG [RS_OPEN_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_OPEN_REGION, pid=139}] regionserver.HRegion(1048): stopping wal replay for 1930901e64343eeb48a7930647197c36 2024-11-23T16:16:08,690 DEBUG [RS_OPEN_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_OPEN_REGION, pid=139}] regionserver.HRegion(1060): Cleaning up temporary data for 1930901e64343eeb48a7930647197c36 2024-11-23T16:16:08,690 DEBUG [RS_OPEN_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_OPEN_REGION, pid=139}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-23T16:16:08,691 DEBUG [RS_OPEN_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_OPEN_REGION, pid=139}] regionserver.HRegion(1093): writing seq id for 1930901e64343eeb48a7930647197c36 2024-11-23T16:16:08,692 INFO [RS_OPEN_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_OPEN_REGION, pid=139}] regionserver.HRegion(1114): Opened 1930901e64343eeb48a7930647197c36; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66852619, jitterRate=-0.0038183480501174927}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-23T16:16:08,692 DEBUG [RS_OPEN_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_OPEN_REGION, pid=139}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1930901e64343eeb48a7930647197c36 2024-11-23T16:16:08,692 DEBUG [RS_OPEN_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_OPEN_REGION, pid=139}] regionserver.HRegion(1006): Region open journal for 1930901e64343eeb48a7930647197c36: Running coprocessor pre-open hook at 1732378568683Writing region info on filesystem at 1732378568683Initializing all the Stores at 1732378568683Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732378568683Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732378568683Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732378568683Cleaning up temporary data from old regions at 1732378568690 (+7 ms)Running coprocessor post-open hooks at 1732378568692 (+2 ms)Region opened successfully at 1732378568692 2024-11-23T16:16:08,693 INFO [RS_OPEN_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_OPEN_REGION, pid=139}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., pid=139, masterSystemTime=1732378568679 2024-11-23T16:16:08,695 DEBUG [RS_OPEN_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_OPEN_REGION, pid=139}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36. 2024-11-23T16:16:08,695 INFO [RS_OPEN_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_OPEN_REGION, pid=139}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36. 2024-11-23T16:16:08,695 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=137 updating hbase:meta row=1930901e64343eeb48a7930647197c36, regionState=OPEN, openSeqNum=5, regionLocation=1a7c5470b472,44715,1732378420375 2024-11-23T16:16:08,696 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=139, ppid=137, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 because future has completed 2024-11-23T16:16:08,699 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=139, resume processing ppid=137 2024-11-23T16:16:08,699 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=139, ppid=137, state=SUCCESS, hasLock=false; OpenRegionProcedure 1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 in 169 msec 2024-11-23T16:16:08,700 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=137, resume processing ppid=136 2024-11-23T16:16:08,700 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=137, ppid=136, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1930901e64343eeb48a7930647197c36, REOPEN/MOVE in 487 msec 2024-11-23T16:16:08,703 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=136, resume processing ppid=135 2024-11-23T16:16:08,703 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=136, ppid=135, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 490 msec 2024-11-23T16:16:08,705 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=135, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 905 msec 2024-11-23T16:16:08,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-23T16:16:08,705 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-23T16:16:08,706 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@53954261, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T16:16:08,706 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1a7c5470b472,45799,-1 for getting cluster id 2024-11-23T16:16:08,706 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-23T16:16:08,707 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '34e0592b-548a-4d73-be6a-df1bafadfd92' 2024-11-23T16:16:08,707 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-23T16:16:08,707 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "34e0592b-548a-4d73-be6a-df1bafadfd92" 2024-11-23T16:16:08,707 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@c5b26d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T16:16:08,708 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1a7c5470b472,45799,-1] 2024-11-23T16:16:08,708 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-23T16:16:08,708 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T16:16:08,709 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36288, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-23T16:16:08,709 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1c56f954, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T16:16:08,710 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@143fb08, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T16:16:08,710 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1a7c5470b472,45799,-1 for getting cluster id 2024-11-23T16:16:08,710 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-23T16:16:08,710 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '34e0592b-548a-4d73-be6a-df1bafadfd92' 2024-11-23T16:16:08,710 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-23T16:16:08,710 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "34e0592b-548a-4d73-be6a-df1bafadfd92" 2024-11-23T16:16:08,711 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@8935a28, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T16:16:08,711 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1a7c5470b472,45799,-1] 2024-11-23T16:16:08,711 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-23T16:16:08,711 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T16:16:08,711 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36310, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-23T16:16:08,712 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32a78359, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T16:16:08,712 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@40503a36, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T16:16:08,712 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1a7c5470b472,45799,-1 for getting cluster id 2024-11-23T16:16:08,713 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-23T16:16:08,713 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '34e0592b-548a-4d73-be6a-df1bafadfd92' 2024-11-23T16:16:08,713 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-23T16:16:08,713 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "34e0592b-548a-4d73-be6a-df1bafadfd92" 2024-11-23T16:16:08,713 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7029f7ac, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T16:16:08,713 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1a7c5470b472,45799,-1] 2024-11-23T16:16:08,713 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-23T16:16:08,714 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T16:16:08,714 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36332, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-23T16:16:08,714 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@30695af, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T16:16:08,715 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@52f72e1a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T16:16:08,715 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1a7c5470b472,45799,-1 for getting cluster id 2024-11-23T16:16:08,715 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-23T16:16:08,716 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '34e0592b-548a-4d73-be6a-df1bafadfd92' 2024-11-23T16:16:08,716 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-23T16:16:08,716 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "34e0592b-548a-4d73-be6a-df1bafadfd92" 2024-11-23T16:16:08,716 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3ab721a3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T16:16:08,716 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1a7c5470b472,45799,-1] 2024-11-23T16:16:08,716 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-23T16:16:08,716 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T16:16:08,717 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36352, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-23T16:16:08,717 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@21ef182f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T16:16:08,718 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@47173f60, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T16:16:08,718 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1a7c5470b472,45799,-1 for getting cluster id 2024-11-23T16:16:08,718 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-23T16:16:08,718 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '34e0592b-548a-4d73-be6a-df1bafadfd92' 2024-11-23T16:16:08,718 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-23T16:16:08,718 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "34e0592b-548a-4d73-be6a-df1bafadfd92" 2024-11-23T16:16:08,719 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4dd49b97, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T16:16:08,719 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1a7c5470b472,45799,-1] 2024-11-23T16:16:08,719 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-23T16:16:08,719 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T16:16:08,719 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36372, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-23T16:16:08,720 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@79ba110d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T16:16:08,720 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7122db2b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T16:16:08,720 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1a7c5470b472,45799,-1 for getting cluster id 2024-11-23T16:16:08,720 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-23T16:16:08,721 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '34e0592b-548a-4d73-be6a-df1bafadfd92' 2024-11-23T16:16:08,721 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-23T16:16:08,721 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "34e0592b-548a-4d73-be6a-df1bafadfd92" 2024-11-23T16:16:08,721 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2506ae16, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T16:16:08,721 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1a7c5470b472,45799,-1] 2024-11-23T16:16:08,721 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-23T16:16:08,721 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T16:16:08,722 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36382, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-23T16:16:08,722 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@555f7aeb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T16:16:08,723 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3666b173, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T16:16:08,723 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1a7c5470b472,45799,-1 for getting cluster id 2024-11-23T16:16:08,723 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-23T16:16:08,724 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '34e0592b-548a-4d73-be6a-df1bafadfd92' 2024-11-23T16:16:08,724 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-23T16:16:08,724 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "34e0592b-548a-4d73-be6a-df1bafadfd92" 2024-11-23T16:16:08,724 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6df1119a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T16:16:08,724 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1a7c5470b472,45799,-1] 2024-11-23T16:16:08,725 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-23T16:16:08,725 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T16:16:08,725 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36396, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-23T16:16:08,726 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4ae2bc7b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T16:16:08,726 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@77ff027c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T16:16:08,726 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1a7c5470b472,45799,-1 for getting cluster id 2024-11-23T16:16:08,726 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-23T16:16:08,727 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '34e0592b-548a-4d73-be6a-df1bafadfd92' 2024-11-23T16:16:08,727 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-23T16:16:08,727 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "34e0592b-548a-4d73-be6a-df1bafadfd92" 2024-11-23T16:16:08,727 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1593222c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T16:16:08,727 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1a7c5470b472,45799,-1] 2024-11-23T16:16:08,728 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-23T16:16:08,728 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T16:16:08,728 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36422, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-23T16:16:08,729 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f0adc43, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T16:16:08,729 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2637386d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T16:16:08,729 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1a7c5470b472,45799,-1 for getting cluster id 2024-11-23T16:16:08,730 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-23T16:16:08,730 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '34e0592b-548a-4d73-be6a-df1bafadfd92' 2024-11-23T16:16:08,730 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-23T16:16:08,730 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "34e0592b-548a-4d73-be6a-df1bafadfd92" 2024-11-23T16:16:08,730 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2af4f195, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T16:16:08,730 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1a7c5470b472,45799,-1] 2024-11-23T16:16:08,730 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-23T16:16:08,731 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T16:16:08,731 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36446, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-23T16:16:08,732 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@cbe0902, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T16:16:08,732 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2455dde6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T16:16:08,732 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1a7c5470b472,45799,-1 for getting cluster id 2024-11-23T16:16:08,732 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-23T16:16:08,733 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '34e0592b-548a-4d73-be6a-df1bafadfd92' 2024-11-23T16:16:08,733 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-23T16:16:08,733 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "34e0592b-548a-4d73-be6a-df1bafadfd92" 2024-11-23T16:16:08,733 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2031f1af, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T16:16:08,733 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1a7c5470b472,45799,-1] 2024-11-23T16:16:08,733 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-23T16:16:08,733 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T16:16:08,734 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36468, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-23T16:16:08,734 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3b5b4949, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T16:16:08,735 DEBUG [Thread-2302 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T16:16:08,735 DEBUG [Thread-2319 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T16:16:08,735 DEBUG [Thread-2317 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T16:16:08,735 DEBUG [Thread-2306 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T16:16:08,735 DEBUG [Thread-2310 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T16:16:08,735 DEBUG [Thread-2321 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T16:16:08,736 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1a7c5470b472,44715,1732378420375, seqNum=-1] 2024-11-23T16:16:08,736 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1a7c5470b472,44715,1732378420375, seqNum=-1] 2024-11-23T16:16:08,736 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1a7c5470b472,44715,1732378420375, seqNum=-1] 2024-11-23T16:16:08,736 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1a7c5470b472,44715,1732378420375, seqNum=-1] 2024-11-23T16:16:08,736 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T16:16:08,736 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T16:16:08,736 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T16:16:08,736 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1a7c5470b472,44715,1732378420375, seqNum=-1] 2024-11-23T16:16:08,736 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T16:16:08,736 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T16:16:08,736 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1a7c5470b472,44715,1732378420375, seqNum=-1] 2024-11-23T16:16:08,737 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45799 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-23T16:16:08,737 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T16:16:08,737 DEBUG [Thread-2308 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T16:16:08,737 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53774, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T16:16:08,737 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53766, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T16:16:08,737 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53800, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T16:16:08,738 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53768, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T16:16:08,738 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53780, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T16:16:08,738 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53788, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T16:16:08,738 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1a7c5470b472,44715,1732378420375, seqNum=-1] 2024-11-23T16:16:08,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45799 {}] procedure2.ProcedureExecutor(1139): Stored pid=140, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=140, table=TestAcidGuarantees 2024-11-23T16:16:08,739 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T16:16:08,739 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5] 2024-11-23T16:16:08,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=140 2024-11-23T16:16:08,740 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=140, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=140, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-23T16:16:08,740 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5] 2024-11-23T16:16:08,740 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5] 2024-11-23T16:16:08,740 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5] 2024-11-23T16:16:08,741 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53810, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T16:16:08,741 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=140, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=140, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-23T16:16:08,742 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=141, ppid=140, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-23T16:16:08,742 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5] 2024-11-23T16:16:08,742 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5] 2024-11-23T16:16:08,742 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5] 2024-11-23T16:16:08,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(8855): Flush requested on 1930901e64343eeb48a7930647197c36 2024-11-23T16:16:08,747 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1930901e64343eeb48a7930647197c36 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-23T16:16:08,748 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1930901e64343eeb48a7930647197c36, store=A 2024-11-23T16:16:08,748 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-23T16:16:08,748 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1930901e64343eeb48a7930647197c36, store=B 2024-11-23T16:16:08,748 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-23T16:16:08,748 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1930901e64343eeb48a7930647197c36, store=C 2024-11-23T16:16:08,748 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-23T16:16:08,749 DEBUG [Thread-2313 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T16:16:08,749 DEBUG [Thread-2315 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T16:16:08,749 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1a7c5470b472,44715,1732378420375, seqNum=-1] 2024-11-23T16:16:08,749 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T16:16:08,749 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1a7c5470b472,44715,1732378420375, seqNum=-1] 2024-11-23T16:16:08,750 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T16:16:08,751 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53812, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T16:16:08,751 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53820, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T16:16:08,752 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5] 2024-11-23T16:16:08,753 DEBUG [Thread-2304 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T16:16:08,753 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5] 2024-11-23T16:16:08,753 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1a7c5470b472,44715,1732378420375, seqNum=-1] 2024-11-23T16:16:08,757 DEBUG [Thread-2304 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T16:16:08,761 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53830, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T16:16:08,768 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5] 2024-11-23T16:16:08,774 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1a7c5470b472:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732378568756, max timestamp=1732378568772Num uniques 859; 2024-11-23T16:16:08,774 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1a7c5470b472:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732378568756, max timestamp=1732378568772Num uniques 859; 2024-11-23T16:16:08,775 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1a7c5470b472:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-23T16:16:08,775 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1a7c5470b472:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-23T16:16:08,775 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1a7c5470b472:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732378568756, max timestamp=1732378568773Num uniques 859; 2024-11-23T16:16:08,775 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1a7c5470b472:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-23T16:16:08,779 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241123ff1b460233594db69b5d5dbbf38ca8bd_1930901e64343eeb48a7930647197c36 is 50, key is test_row_1/A:col10/1732378568747/Put/seqid=0 2024-11-23T16:16:08,790 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742339_1515 (size=9714) 2024-11-23T16:16:08,796 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:16:08,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53810 deadline: 1732378628790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:16:08,796 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:08,796 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:08,796 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:16:08,796 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:16:08,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53774 deadline: 1732378628791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:16:08,797 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:08,797 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:08,797 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:16:08,797 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:16:08,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53830 deadline: 1732378628792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:16:08,797 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:08,797 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:08,797 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:16:08,798 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:16:08,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53788 deadline: 1732378628793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:16:08,798 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:08,798 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:08,798 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:16:08,798 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:16:08,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53766 deadline: 1732378628795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:16:08,799 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:08,799 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:08,799 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:16:08,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=140 2024-11-23T16:16:08,896 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=141 2024-11-23T16:16:08,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36. 2024-11-23T16:16:08,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36. as already flushing 2024-11-23T16:16:08,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36. 2024-11-23T16:16:08,896 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] handler.RSProcedureHandler(58): pid=141 java.io.IOException: Unable to complete flush {ENCODED => 1930901e64343eeb48a7930647197c36, NAME => 'TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:16:08,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=141 java.io.IOException: Unable to complete flush {ENCODED => 1930901e64343eeb48a7930647197c36, NAME => 'TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:16:08,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45799 {}] master.HMaster(4177): Remote procedure failed, pid=141 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1930901e64343eeb48a7930647197c36, NAME => 'TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1930901e64343eeb48a7930647197c36, NAME => 'TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:16:08,911 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:16:08,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53810 deadline: 1732378628907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:16:08,912 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:16:08,912 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:08,912 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:08,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53774 deadline: 1732378628907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:16:08,912 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:16:08,912 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:08,912 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:16:08,912 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:08,912 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:16:08,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53830 deadline: 1732378628908, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:16:08,912 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:16:08,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53788 deadline: 1732378628908, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:16:08,912 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:16:08,913 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:08,913 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:08,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53766 deadline: 1732378628908, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:16:08,913 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:08,913 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:16:08,913 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:08,913 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:16:08,913 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:08,913 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:08,913 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:16:09,049 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=141 2024-11-23T16:16:09,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36. 2024-11-23T16:16:09,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36. as already flushing 2024-11-23T16:16:09,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36. 2024-11-23T16:16:09,049 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] handler.RSProcedureHandler(58): pid=141 java.io.IOException: Unable to complete flush {ENCODED => 1930901e64343eeb48a7930647197c36, NAME => 'TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:16:09,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=141 java.io.IOException: Unable to complete flush {ENCODED => 1930901e64343eeb48a7930647197c36, NAME => 'TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:16:09,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45799 {}] master.HMaster(4177): Remote procedure failed, pid=141 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1930901e64343eeb48a7930647197c36, NAME => 'TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1930901e64343eeb48a7930647197c36, NAME => 'TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:16:09,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=140 2024-11-23T16:16:09,120 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:16:09,120 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:16:09,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53774 deadline: 1732378629118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:16:09,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53810 deadline: 1732378629117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:16:09,120 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:16:09,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53830 deadline: 1732378629118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:16:09,120 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:09,120 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:09,120 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:09,120 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:09,120 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:16:09,120 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:16:09,121 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:16:09,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53788 deadline: 1732378629118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:16:09,121 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:09,121 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:09,121 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:16:09,121 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:16:09,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53766 deadline: 1732378629118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:16:09,121 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:09,121 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:09,121 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:16:09,121 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:09,121 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:09,121 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:16:09,190 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:09,194 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241123ff1b460233594db69b5d5dbbf38ca8bd_1930901e64343eeb48a7930647197c36 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123ff1b460233594db69b5d5dbbf38ca8bd_1930901e64343eeb48a7930647197c36 2024-11-23T16:16:09,195 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/.tmp/A/b1c124a4b0a34192aaaace777e6da0f1, store: [table=TestAcidGuarantees family=A region=1930901e64343eeb48a7930647197c36] 2024-11-23T16:16:09,196 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/.tmp/A/b1c124a4b0a34192aaaace777e6da0f1 is 175, key is test_row_1/A:col10/1732378568747/Put/seqid=0 2024-11-23T16:16:09,200 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742340_1516 (size=22361) 2024-11-23T16:16:09,201 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=15, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/.tmp/A/b1c124a4b0a34192aaaace777e6da0f1 2024-11-23T16:16:09,202 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=141 2024-11-23T16:16:09,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36. 2024-11-23T16:16:09,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36. as already flushing 2024-11-23T16:16:09,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36. 2024-11-23T16:16:09,202 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] handler.RSProcedureHandler(58): pid=141 java.io.IOException: Unable to complete flush {ENCODED => 1930901e64343eeb48a7930647197c36, NAME => 'TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:16:09,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=141 java.io.IOException: Unable to complete flush {ENCODED => 1930901e64343eeb48a7930647197c36, NAME => 'TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:16:09,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45799 {}] master.HMaster(4177): Remote procedure failed, pid=141 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1930901e64343eeb48a7930647197c36, NAME => 'TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1930901e64343eeb48a7930647197c36, NAME => 'TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:16:09,227 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/.tmp/B/daa00491345a4740bf7938eed86e2301 is 50, key is test_row_1/B:col10/1732378568747/Put/seqid=0 2024-11-23T16:16:09,231 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742341_1517 (size=9657) 2024-11-23T16:16:09,231 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-23T16:16:09,233 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/.tmp/B/daa00491345a4740bf7938eed86e2301 2024-11-23T16:16:09,264 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/.tmp/C/d821084003594d5fae0df646a986689c is 50, key is test_row_1/C:col10/1732378568747/Put/seqid=0 2024-11-23T16:16:09,268 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742342_1518 (size=9657) 2024-11-23T16:16:09,355 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=141 2024-11-23T16:16:09,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36. 2024-11-23T16:16:09,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36. as already flushing 2024-11-23T16:16:09,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36. 2024-11-23T16:16:09,355 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] handler.RSProcedureHandler(58): pid=141 java.io.IOException: Unable to complete flush {ENCODED => 1930901e64343eeb48a7930647197c36, NAME => 'TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:16:09,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=141 java.io.IOException: Unable to complete flush {ENCODED => 1930901e64343eeb48a7930647197c36, NAME => 'TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:16:09,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45799 {}] master.HMaster(4177): Remote procedure failed, pid=141 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1930901e64343eeb48a7930647197c36, NAME => 'TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1930901e64343eeb48a7930647197c36, NAME => 'TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:16:09,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=140 2024-11-23T16:16:09,431 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:16:09,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53810 deadline: 1732378629428, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:16:09,431 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:16:09,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53788 deadline: 1732378629428, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:16:09,431 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:16:09,431 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:09,432 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:09,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53774 deadline: 1732378629428, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:16:09,432 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:09,432 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:09,432 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:16:09,432 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:16:09,432 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:16:09,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53830 deadline: 1732378629428, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:16:09,432 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:09,432 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:09,432 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:16:09,432 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:16:09,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53766 deadline: 1732378629428, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:16:09,432 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:09,432 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:09,432 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:16:09,432 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:09,432 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:09,432 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:16:09,508 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=141 2024-11-23T16:16:09,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36. 2024-11-23T16:16:09,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36. as already flushing 2024-11-23T16:16:09,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36. 2024-11-23T16:16:09,508 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] handler.RSProcedureHandler(58): pid=141 java.io.IOException: Unable to complete flush {ENCODED => 1930901e64343eeb48a7930647197c36, NAME => 'TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:16:09,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=141 java.io.IOException: Unable to complete flush {ENCODED => 1930901e64343eeb48a7930647197c36, NAME => 'TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:16:09,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45799 {}] master.HMaster(4177): Remote procedure failed, pid=141 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1930901e64343eeb48a7930647197c36, NAME => 'TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1930901e64343eeb48a7930647197c36, NAME => 'TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:16:09,661 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=141 2024-11-23T16:16:09,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36. 2024-11-23T16:16:09,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36. as already flushing 2024-11-23T16:16:09,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36. 2024-11-23T16:16:09,661 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] handler.RSProcedureHandler(58): pid=141 java.io.IOException: Unable to complete flush {ENCODED => 1930901e64343eeb48a7930647197c36, NAME => 'TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:16:09,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=141 java.io.IOException: Unable to complete flush {ENCODED => 1930901e64343eeb48a7930647197c36, NAME => 'TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:16:09,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45799 {}] master.HMaster(4177): Remote procedure failed, pid=141 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1930901e64343eeb48a7930647197c36, NAME => 'TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1930901e64343eeb48a7930647197c36, NAME => 'TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:16:09,668 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/.tmp/C/d821084003594d5fae0df646a986689c 2024-11-23T16:16:09,672 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/.tmp/A/b1c124a4b0a34192aaaace777e6da0f1 as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/A/b1c124a4b0a34192aaaace777e6da0f1 2024-11-23T16:16:09,679 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/A/b1c124a4b0a34192aaaace777e6da0f1, entries=100, sequenceid=15, filesize=21.8 K 2024-11-23T16:16:09,680 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/.tmp/B/daa00491345a4740bf7938eed86e2301 as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/B/daa00491345a4740bf7938eed86e2301 2024-11-23T16:16:09,683 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/B/daa00491345a4740bf7938eed86e2301, entries=100, sequenceid=15, filesize=9.4 K 2024-11-23T16:16:09,684 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/.tmp/C/d821084003594d5fae0df646a986689c as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/C/d821084003594d5fae0df646a986689c 2024-11-23T16:16:09,687 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/C/d821084003594d5fae0df646a986689c, entries=100, sequenceid=15, filesize=9.4 K 2024-11-23T16:16:09,689 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 1930901e64343eeb48a7930647197c36 in 943ms, sequenceid=15, compaction requested=false 2024-11-23T16:16:09,689 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1930901e64343eeb48a7930647197c36: 2024-11-23T16:16:09,813 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=141 2024-11-23T16:16:09,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36. 2024-11-23T16:16:09,814 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegion(2902): Flushing 1930901e64343eeb48a7930647197c36 3/3 column families, dataSize=127.47 KB heapSize=365.06 KB 2024-11-23T16:16:09,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1930901e64343eeb48a7930647197c36, store=A 2024-11-23T16:16:09,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-23T16:16:09,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1930901e64343eeb48a7930647197c36, store=B 2024-11-23T16:16:09,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-23T16:16:09,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1930901e64343eeb48a7930647197c36, store=C 2024-11-23T16:16:09,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-23T16:16:09,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411235fbe8cf7f8a14ea585a13e4c516cf298_1930901e64343eeb48a7930647197c36 is 50, key is test_row_0/A:col10/1732378568788/Put/seqid=0 2024-11-23T16:16:09,825 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742343_1519 (size=12154) 2024-11-23T16:16:09,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=140 2024-11-23T16:16:09,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(8855): Flush requested on 1930901e64343eeb48a7930647197c36 2024-11-23T16:16:09,942 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36. as already flushing 2024-11-23T16:16:09,951 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:16:09,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53766 deadline: 1732378629944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:16:09,951 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:09,952 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:09,952 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:16:09,956 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:16:09,956 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:16:09,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53774 deadline: 1732378629950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:16:09,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53788 deadline: 1732378629950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:16:09,957 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:09,957 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:16:09,957 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:09,957 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:16:09,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53810 deadline: 1732378629951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:16:09,957 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:16:09,957 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:09,957 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:09,957 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:09,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53830 deadline: 1732378629952, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:16:09,957 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:09,957 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:16:09,957 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:16:09,958 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:09,958 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:09,958 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:16:10,062 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:16:10,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53766 deadline: 1732378630057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:16:10,063 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:10,063 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:10,063 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:16:10,070 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:16:10,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53774 deadline: 1732378630068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:16:10,071 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:16:10,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53788 deadline: 1732378630068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:16:10,071 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:10,071 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:10,071 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:16:10,071 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:16:10,071 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:10,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53810 deadline: 1732378630068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:16:10,071 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:10,071 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:16:10,071 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:16:10,071 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:10,071 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:10,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53830 deadline: 1732378630068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:16:10,071 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:16:10,072 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:10,072 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:10,072 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:16:10,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:10,229 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411235fbe8cf7f8a14ea585a13e4c516cf298_1930901e64343eeb48a7930647197c36 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411235fbe8cf7f8a14ea585a13e4c516cf298_1930901e64343eeb48a7930647197c36 2024-11-23T16:16:10,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/.tmp/A/19b2fe1b7c5a42cf9a642083b4edf826, store: [table=TestAcidGuarantees family=A region=1930901e64343eeb48a7930647197c36] 2024-11-23T16:16:10,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/.tmp/A/19b2fe1b7c5a42cf9a642083b4edf826 is 175, key is test_row_0/A:col10/1732378568788/Put/seqid=0 2024-11-23T16:16:10,239 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742344_1520 (size=30955) 2024-11-23T16:16:10,271 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:16:10,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53766 deadline: 1732378630268, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:16:10,271 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:10,271 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:10,271 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:16:10,282 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:16:10,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53774 deadline: 1732378630278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:16:10,282 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:10,282 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:16:10,282 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:10,282 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:16:10,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53788 deadline: 1732378630278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:16:10,282 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:16:10,283 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:10,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53810 deadline: 1732378630278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:16:10,283 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:10,283 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:16:10,283 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:16:10,283 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:10,283 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:10,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53830 deadline: 1732378630278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:16:10,283 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:16:10,283 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:10,283 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:10,283 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:16:10,581 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:16:10,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53766 deadline: 1732378630577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:16:10,581 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:10,581 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:10,581 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:16:10,591 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:16:10,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53774 deadline: 1732378630588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:16:10,591 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:16:10,591 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:16:10,592 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:10,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53810 deadline: 1732378630588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:16:10,592 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:10,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53788 deadline: 1732378630588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:16:10,592 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:16:10,592 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:10,592 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:16:10,592 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:10,592 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:10,592 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:10,592 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:16:10,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53830 deadline: 1732378630588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:16:10,592 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:16:10,592 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:10,592 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:10,592 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:16:10,639 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=51, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/.tmp/A/19b2fe1b7c5a42cf9a642083b4edf826 2024-11-23T16:16:10,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/.tmp/B/34d1b0a989f14c419717ba7da246b8db is 50, key is test_row_0/B:col10/1732378568788/Put/seqid=0 2024-11-23T16:16:10,662 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742345_1521 (size=12001) 2024-11-23T16:16:10,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=140 2024-11-23T16:16:11,063 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/.tmp/B/34d1b0a989f14c419717ba7da246b8db 2024-11-23T16:16:11,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/.tmp/C/66b396aa48af4c668f52e8c6f3905c1b is 50, key is test_row_0/C:col10/1732378568788/Put/seqid=0 2024-11-23T16:16:11,091 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:16:11,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53766 deadline: 1732378631088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:16:11,092 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:11,092 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:11,092 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:16:11,094 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742346_1522 (size=12001) 2024-11-23T16:16:11,096 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/.tmp/C/66b396aa48af4c668f52e8c6f3905c1b 2024-11-23T16:16:11,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/.tmp/A/19b2fe1b7c5a42cf9a642083b4edf826 as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/A/19b2fe1b7c5a42cf9a642083b4edf826 2024-11-23T16:16:11,102 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:16:11,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53774 deadline: 1732378631098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:16:11,102 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:16:11,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53810 deadline: 1732378631098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:16:11,103 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:11,103 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:11,103 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:16:11,103 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:16:11,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53788 deadline: 1732378631098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:16:11,103 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:11,103 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:11,103 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:16:11,103 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:16:11,103 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:11,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53830 deadline: 1732378631098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:16:11,103 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:11,103 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:16:11,103 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:11,103 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:11,103 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:16:11,104 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/A/19b2fe1b7c5a42cf9a642083b4edf826, entries=150, sequenceid=51, filesize=30.2 K 2024-11-23T16:16:11,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/.tmp/B/34d1b0a989f14c419717ba7da246b8db as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/B/34d1b0a989f14c419717ba7da246b8db 2024-11-23T16:16:11,117 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/B/34d1b0a989f14c419717ba7da246b8db, entries=150, sequenceid=51, filesize=11.7 K 2024-11-23T16:16:11,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/.tmp/C/66b396aa48af4c668f52e8c6f3905c1b as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/C/66b396aa48af4c668f52e8c6f3905c1b 2024-11-23T16:16:11,129 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/C/66b396aa48af4c668f52e8c6f3905c1b, entries=150, sequenceid=51, filesize=11.7 K 2024-11-23T16:16:11,130 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~365.02 KB/373776, currentSize=60.38 KB/61830 for 1930901e64343eeb48a7930647197c36 in 1316ms, sequenceid=51, compaction requested=false 2024-11-23T16:16:11,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegion(2603): Flush status journal for 1930901e64343eeb48a7930647197c36: 2024-11-23T16:16:11,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36. 2024-11-23T16:16:11,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=141 2024-11-23T16:16:11,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45799 {}] master.HMaster(4169): Remote procedure done, pid=141 2024-11-23T16:16:11,135 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=141, resume processing ppid=140 2024-11-23T16:16:11,135 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=141, ppid=140, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3910 sec 2024-11-23T16:16:11,137 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=140, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=140, table=TestAcidGuarantees in 2.3980 sec 2024-11-23T16:16:12,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(8855): Flush requested on 1930901e64343eeb48a7930647197c36 2024-11-23T16:16:12,110 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1930901e64343eeb48a7930647197c36 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-23T16:16:12,111 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1930901e64343eeb48a7930647197c36, store=A 2024-11-23T16:16:12,111 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-23T16:16:12,111 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1930901e64343eeb48a7930647197c36, store=B 2024-11-23T16:16:12,111 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-23T16:16:12,111 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1930901e64343eeb48a7930647197c36, store=C 2024-11-23T16:16:12,111 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-23T16:16:12,117 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411232d88340642604ed19002eafb984debe0_1930901e64343eeb48a7930647197c36 is 50, key is test_row_0/A:col10/1732378569950/Put/seqid=0 2024-11-23T16:16:12,122 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742347_1523 (size=12154) 2024-11-23T16:16:12,133 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:16:12,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53810 deadline: 1732378632126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:16:12,133 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:16:12,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53830 deadline: 1732378632127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:16:12,133 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:12,133 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:12,133 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:16:12,133 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:12,133 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:12,133 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:16:12,133 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:16:12,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53774 deadline: 1732378632130, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:16:12,134 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:12,134 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:12,134 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:16:12,137 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:16:12,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53788 deadline: 1732378632132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:16:12,137 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:16:12,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53766 deadline: 1732378632133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:16:12,137 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:12,137 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:12,138 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:16:12,138 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:12,138 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:12,138 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:16:12,241 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:16:12,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53810 deadline: 1732378632238, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:16:12,241 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:16:12,241 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:12,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53830 deadline: 1732378632238, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:16:12,242 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:12,242 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:16:12,242 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:12,242 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:16:12,242 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:12,242 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:16:12,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53774 deadline: 1732378632238, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:16:12,242 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:12,242 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:12,242 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:16:12,250 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:16:12,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53788 deadline: 1732378632248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:16:12,250 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:12,250 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:12,250 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:16:12,252 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:16:12,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53766 deadline: 1732378632249, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:16:12,252 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:12,252 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:12,252 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:16:12,451 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:16:12,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53810 deadline: 1732378632448, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:16:12,452 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:16:12,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53830 deadline: 1732378632448, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:16:12,452 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:16:12,452 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:12,452 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:12,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53774 deadline: 1732378632448, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:16:12,452 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:16:12,452 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:12,452 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:12,452 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:16:12,452 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:12,452 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:12,452 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:16:12,460 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:16:12,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53788 deadline: 1732378632457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:16:12,461 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:12,461 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:16:12,461 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:12,461 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:16:12,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53766 deadline: 1732378632458, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:16:12,461 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:12,461 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:12,461 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:16:12,523 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:12,526 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411232d88340642604ed19002eafb984debe0_1930901e64343eeb48a7930647197c36 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411232d88340642604ed19002eafb984debe0_1930901e64343eeb48a7930647197c36 2024-11-23T16:16:12,527 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/.tmp/A/d1ab2ef581cc419bb126c47d56b4e937, store: [table=TestAcidGuarantees family=A region=1930901e64343eeb48a7930647197c36] 2024-11-23T16:16:12,528 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/.tmp/A/d1ab2ef581cc419bb126c47d56b4e937 is 175, key is test_row_0/A:col10/1732378569950/Put/seqid=0 2024-11-23T16:16:12,534 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742348_1524 (size=30955) 2024-11-23T16:16:12,761 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:16:12,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53810 deadline: 1732378632758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:16:12,761 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:16:12,761 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:12,761 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:12,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53830 deadline: 1732378632758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:16:12,761 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:16:12,761 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:16:12,761 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:12,762 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:12,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53774 deadline: 1732378632758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:16:12,762 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:16:12,762 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:12,762 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:12,762 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:16:12,770 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:16:12,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53788 deadline: 1732378632768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:16:12,770 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:16:12,770 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:12,770 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:12,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53766 deadline: 1732378632768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:16:12,770 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:16:12,770 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:12,770 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:12,770 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:16:12,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=140 2024-11-23T16:16:12,898 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-23T16:16:12,898 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45799 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-23T16:16:12,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45799 {}] procedure2.ProcedureExecutor(1139): Stored pid=142, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=142, table=TestAcidGuarantees 2024-11-23T16:16:12,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-23T16:16:12,900 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=142, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=142, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-23T16:16:12,901 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=142, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=142, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-23T16:16:12,901 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=143, ppid=142, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-23T16:16:12,935 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=65, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/.tmp/A/d1ab2ef581cc419bb126c47d56b4e937 2024-11-23T16:16:12,941 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/.tmp/B/26ba178db8a54e50a6d897c63d5aa5ec is 50, key is test_row_0/B:col10/1732378569950/Put/seqid=0 2024-11-23T16:16:12,944 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742349_1525 (size=12001) 2024-11-23T16:16:13,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-23T16:16:13,053 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=143 2024-11-23T16:16:13,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36. 2024-11-23T16:16:13,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36. as already flushing 2024-11-23T16:16:13,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36. 2024-11-23T16:16:13,053 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] handler.RSProcedureHandler(58): pid=143 java.io.IOException: Unable to complete flush {ENCODED => 1930901e64343eeb48a7930647197c36, NAME => 'TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:16:13,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=143 java.io.IOException: Unable to complete flush {ENCODED => 1930901e64343eeb48a7930647197c36, NAME => 'TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:16:13,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45799 {}] master.HMaster(4177): Remote procedure failed, pid=143 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1930901e64343eeb48a7930647197c36, NAME => 'TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1930901e64343eeb48a7930647197c36, NAME => 'TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:16:13,206 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=143 2024-11-23T16:16:13,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36. 2024-11-23T16:16:13,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36. as already flushing 2024-11-23T16:16:13,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36. 2024-11-23T16:16:13,206 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] handler.RSProcedureHandler(58): pid=143 java.io.IOException: Unable to complete flush {ENCODED => 1930901e64343eeb48a7930647197c36, NAME => 'TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:16:13,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=143 java.io.IOException: Unable to complete flush {ENCODED => 1930901e64343eeb48a7930647197c36, NAME => 'TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:16:13,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45799 {}] master.HMaster(4177): Remote procedure failed, pid=143 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1930901e64343eeb48a7930647197c36, NAME => 'TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1930901e64343eeb48a7930647197c36, NAME => 'TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:16:13,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-23T16:16:13,270 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:16:13,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53810 deadline: 1732378633267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:16:13,271 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:13,271 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:16:13,271 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:13,271 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:16:13,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53830 deadline: 1732378633267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:16:13,271 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:16:13,271 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:13,271 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:13,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53774 deadline: 1732378633268, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:16:13,271 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:16:13,271 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:13,271 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:13,271 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:16:13,281 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:16:13,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53788 deadline: 1732378633278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:16:13,282 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:13,282 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:13,282 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:16:13,282 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:16:13,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53766 deadline: 1732378633278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:16:13,282 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:13,282 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:13,282 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:16:13,345 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=65 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/.tmp/B/26ba178db8a54e50a6d897c63d5aa5ec 2024-11-23T16:16:13,352 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/.tmp/C/d8b90a5980094068899bf16be1cf178b is 50, key is test_row_0/C:col10/1732378569950/Put/seqid=0 2024-11-23T16:16:13,357 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742350_1526 (size=12001) 2024-11-23T16:16:13,359 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=143 2024-11-23T16:16:13,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36. 2024-11-23T16:16:13,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36. as already flushing 2024-11-23T16:16:13,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36. 2024-11-23T16:16:13,359 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] handler.RSProcedureHandler(58): pid=143 java.io.IOException: Unable to complete flush {ENCODED => 1930901e64343eeb48a7930647197c36, NAME => 'TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:16:13,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=143 java.io.IOException: Unable to complete flush {ENCODED => 1930901e64343eeb48a7930647197c36, NAME => 'TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:16:13,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45799 {}] master.HMaster(4177): Remote procedure failed, pid=143 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1930901e64343eeb48a7930647197c36, NAME => 'TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1930901e64343eeb48a7930647197c36, NAME => 'TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:16:13,512 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=143 2024-11-23T16:16:13,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36. 2024-11-23T16:16:13,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36. as already flushing 2024-11-23T16:16:13,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36. 2024-11-23T16:16:13,512 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] handler.RSProcedureHandler(58): pid=143 java.io.IOException: Unable to complete flush {ENCODED => 1930901e64343eeb48a7930647197c36, NAME => 'TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:16:13,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=143 java.io.IOException: Unable to complete flush {ENCODED => 1930901e64343eeb48a7930647197c36, NAME => 'TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:16:13,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45799 {}] master.HMaster(4177): Remote procedure failed, pid=143 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1930901e64343eeb48a7930647197c36, NAME => 'TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1930901e64343eeb48a7930647197c36, NAME => 'TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:16:13,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-23T16:16:13,665 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=143 2024-11-23T16:16:13,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36. 2024-11-23T16:16:13,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36. as already flushing 2024-11-23T16:16:13,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36. 2024-11-23T16:16:13,665 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] handler.RSProcedureHandler(58): pid=143 java.io.IOException: Unable to complete flush {ENCODED => 1930901e64343eeb48a7930647197c36, NAME => 'TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:16:13,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=143 java.io.IOException: Unable to complete flush {ENCODED => 1930901e64343eeb48a7930647197c36, NAME => 'TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:16:13,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45799 {}] master.HMaster(4177): Remote procedure failed, pid=143 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1930901e64343eeb48a7930647197c36, NAME => 'TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1930901e64343eeb48a7930647197c36, NAME => 'TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:16:13,758 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=65 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/.tmp/C/d8b90a5980094068899bf16be1cf178b 2024-11-23T16:16:13,762 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/.tmp/A/d1ab2ef581cc419bb126c47d56b4e937 as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/A/d1ab2ef581cc419bb126c47d56b4e937 2024-11-23T16:16:13,765 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/A/d1ab2ef581cc419bb126c47d56b4e937, entries=150, sequenceid=65, filesize=30.2 K 2024-11-23T16:16:13,765 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/.tmp/B/26ba178db8a54e50a6d897c63d5aa5ec as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/B/26ba178db8a54e50a6d897c63d5aa5ec 2024-11-23T16:16:13,769 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/B/26ba178db8a54e50a6d897c63d5aa5ec, entries=150, sequenceid=65, filesize=11.7 K 2024-11-23T16:16:13,769 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/.tmp/C/d8b90a5980094068899bf16be1cf178b as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/C/d8b90a5980094068899bf16be1cf178b 2024-11-23T16:16:13,772 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/C/d8b90a5980094068899bf16be1cf178b, entries=150, sequenceid=65, filesize=11.7 K 2024-11-23T16:16:13,773 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 1930901e64343eeb48a7930647197c36 in 1663ms, sequenceid=65, compaction requested=true 2024-11-23T16:16:13,773 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1930901e64343eeb48a7930647197c36: 2024-11-23T16:16:13,773 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1930901e64343eeb48a7930647197c36:A, priority=-2147483648, current under compaction store size is 1 2024-11-23T16:16:13,773 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T16:16:13,773 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1930901e64343eeb48a7930647197c36:B, priority=-2147483648, current under compaction store size is 2 2024-11-23T16:16:13,773 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T16:16:13,773 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1930901e64343eeb48a7930647197c36:C, priority=-2147483648, current under compaction store size is 3 2024-11-23T16:16:13,773 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T16:16:13,773 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T16:16:13,773 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T16:16:13,774 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T16:16:13,774 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 84271 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T16:16:13,774 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HStore(1541): 1930901e64343eeb48a7930647197c36/B is initiating minor compaction (all files) 2024-11-23T16:16:13,774 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HStore(1541): 1930901e64343eeb48a7930647197c36/A is initiating minor compaction (all files) 2024-11-23T16:16:13,774 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1930901e64343eeb48a7930647197c36/B in TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36. 2024-11-23T16:16:13,774 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1930901e64343eeb48a7930647197c36/A in TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36. 2024-11-23T16:16:13,774 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/B/daa00491345a4740bf7938eed86e2301, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/B/34d1b0a989f14c419717ba7da246b8db, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/B/26ba178db8a54e50a6d897c63d5aa5ec] into tmpdir=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/.tmp, totalSize=32.9 K 2024-11-23T16:16:13,774 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/A/b1c124a4b0a34192aaaace777e6da0f1, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/A/19b2fe1b7c5a42cf9a642083b4edf826, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/A/d1ab2ef581cc419bb126c47d56b4e937] into tmpdir=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/.tmp, totalSize=82.3 K 2024-11-23T16:16:13,774 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36. 2024-11-23T16:16:13,774 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36. files: [hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/A/b1c124a4b0a34192aaaace777e6da0f1, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/A/19b2fe1b7c5a42cf9a642083b4edf826, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/A/d1ab2ef581cc419bb126c47d56b4e937] 2024-11-23T16:16:13,774 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.Compactor(225): Compacting daa00491345a4740bf7938eed86e2301, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1732378568744 2024-11-23T16:16:13,774 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.Compactor(225): Compacting b1c124a4b0a34192aaaace777e6da0f1, keycount=100, bloomtype=ROW, size=21.8 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1732378568744 2024-11-23T16:16:13,775 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.Compactor(225): Compacting 34d1b0a989f14c419717ba7da246b8db, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1732378568784 2024-11-23T16:16:13,775 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.Compactor(225): Compacting 19b2fe1b7c5a42cf9a642083b4edf826, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1732378568784 2024-11-23T16:16:13,775 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.Compactor(225): Compacting 26ba178db8a54e50a6d897c63d5aa5ec, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=65, earliestPutTs=1732378569950 2024-11-23T16:16:13,775 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.Compactor(225): Compacting d1ab2ef581cc419bb126c47d56b4e937, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=65, earliestPutTs=1732378569950 2024-11-23T16:16:13,782 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1930901e64343eeb48a7930647197c36#B#compaction#446 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T16:16:13,783 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/.tmp/B/1c152fa1818f41ada2915546d5afae8c is 50, key is test_row_0/B:col10/1732378569950/Put/seqid=0 2024-11-23T16:16:13,784 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=1930901e64343eeb48a7930647197c36] 2024-11-23T16:16:13,786 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241123ba1618ebe710465a913d4efb84febb1b_1930901e64343eeb48a7930647197c36 store=[table=TestAcidGuarantees family=A region=1930901e64343eeb48a7930647197c36] 2024-11-23T16:16:13,787 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241123ba1618ebe710465a913d4efb84febb1b_1930901e64343eeb48a7930647197c36, store=[table=TestAcidGuarantees family=A region=1930901e64343eeb48a7930647197c36] 2024-11-23T16:16:13,787 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241123ba1618ebe710465a913d4efb84febb1b_1930901e64343eeb48a7930647197c36 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=1930901e64343eeb48a7930647197c36] 2024-11-23T16:16:13,789 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742351_1527 (size=12104) 2024-11-23T16:16:13,792 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742352_1528 (size=4469) 2024-11-23T16:16:13,793 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1930901e64343eeb48a7930647197c36#A#compaction#447 average throughput is 2.71 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T16:16:13,793 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/.tmp/A/9032ef42731641baba6d8904f48110c4 is 175, key is test_row_0/A:col10/1732378569950/Put/seqid=0 2024-11-23T16:16:13,795 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/.tmp/B/1c152fa1818f41ada2915546d5afae8c as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/B/1c152fa1818f41ada2915546d5afae8c 2024-11-23T16:16:13,797 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742353_1529 (size=31058) 2024-11-23T16:16:13,800 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1930901e64343eeb48a7930647197c36/B of 1930901e64343eeb48a7930647197c36 into 1c152fa1818f41ada2915546d5afae8c(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T16:16:13,800 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1930901e64343eeb48a7930647197c36: 2024-11-23T16:16:13,800 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., storeName=1930901e64343eeb48a7930647197c36/B, priority=13, startTime=1732378573773; duration=0sec 2024-11-23T16:16:13,800 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T16:16:13,800 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1930901e64343eeb48a7930647197c36:B 2024-11-23T16:16:13,800 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T16:16:13,802 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/.tmp/A/9032ef42731641baba6d8904f48110c4 as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/A/9032ef42731641baba6d8904f48110c4 2024-11-23T16:16:13,802 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T16:16:13,802 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HStore(1541): 1930901e64343eeb48a7930647197c36/C is initiating minor compaction (all files) 2024-11-23T16:16:13,802 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1930901e64343eeb48a7930647197c36/C in TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36. 2024-11-23T16:16:13,802 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/C/d821084003594d5fae0df646a986689c, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/C/66b396aa48af4c668f52e8c6f3905c1b, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/C/d8b90a5980094068899bf16be1cf178b] into tmpdir=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/.tmp, totalSize=32.9 K 2024-11-23T16:16:13,802 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.Compactor(225): Compacting d821084003594d5fae0df646a986689c, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1732378568744 2024-11-23T16:16:13,803 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.Compactor(225): Compacting 66b396aa48af4c668f52e8c6f3905c1b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1732378568784 2024-11-23T16:16:13,803 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.Compactor(225): Compacting d8b90a5980094068899bf16be1cf178b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=65, earliestPutTs=1732378569950 2024-11-23T16:16:13,806 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1930901e64343eeb48a7930647197c36/A of 1930901e64343eeb48a7930647197c36 into 9032ef42731641baba6d8904f48110c4(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T16:16:13,806 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1930901e64343eeb48a7930647197c36: 2024-11-23T16:16:13,806 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., storeName=1930901e64343eeb48a7930647197c36/A, priority=13, startTime=1732378573773; duration=0sec 2024-11-23T16:16:13,806 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T16:16:13,806 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1930901e64343eeb48a7930647197c36:A 2024-11-23T16:16:13,810 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1930901e64343eeb48a7930647197c36#C#compaction#448 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T16:16:13,810 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/.tmp/C/18f634731ea24c58b1b2b3c86fcb1bf6 is 50, key is test_row_0/C:col10/1732378569950/Put/seqid=0 2024-11-23T16:16:13,813 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742354_1530 (size=12104) 2024-11-23T16:16:13,818 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=143 2024-11-23T16:16:13,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36. 2024-11-23T16:16:13,818 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(2902): Flushing 1930901e64343eeb48a7930647197c36 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-23T16:16:13,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1930901e64343eeb48a7930647197c36, store=A 2024-11-23T16:16:13,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-23T16:16:13,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1930901e64343eeb48a7930647197c36, store=B 2024-11-23T16:16:13,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-23T16:16:13,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1930901e64343eeb48a7930647197c36, store=C 2024-11-23T16:16:13,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-23T16:16:13,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411230b578efdb7b44b039bf6833dd0e0e78c_1930901e64343eeb48a7930647197c36 is 50, key is test_row_0/A:col10/1732378572131/Put/seqid=0 2024-11-23T16:16:13,828 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742355_1531 (size=12154) 2024-11-23T16:16:14,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-23T16:16:14,184 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-23T16:16:14,217 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/.tmp/C/18f634731ea24c58b1b2b3c86fcb1bf6 as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/C/18f634731ea24c58b1b2b3c86fcb1bf6 2024-11-23T16:16:14,221 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1930901e64343eeb48a7930647197c36/C of 1930901e64343eeb48a7930647197c36 into 18f634731ea24c58b1b2b3c86fcb1bf6(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T16:16:14,221 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1930901e64343eeb48a7930647197c36: 2024-11-23T16:16:14,221 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., storeName=1930901e64343eeb48a7930647197c36/C, priority=13, startTime=1732378573773; duration=0sec 2024-11-23T16:16:14,221 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T16:16:14,221 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1930901e64343eeb48a7930647197c36:C 2024-11-23T16:16:14,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:14,231 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411230b578efdb7b44b039bf6833dd0e0e78c_1930901e64343eeb48a7930647197c36 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411230b578efdb7b44b039bf6833dd0e0e78c_1930901e64343eeb48a7930647197c36 2024-11-23T16:16:14,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/.tmp/A/7a7b39e3beb34eb08f2acffbbfa21342, store: [table=TestAcidGuarantees family=A region=1930901e64343eeb48a7930647197c36] 2024-11-23T16:16:14,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/.tmp/A/7a7b39e3beb34eb08f2acffbbfa21342 is 175, key is test_row_0/A:col10/1732378572131/Put/seqid=0 2024-11-23T16:16:14,236 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742356_1532 (size=30955) 2024-11-23T16:16:14,237 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=85, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/.tmp/A/7a7b39e3beb34eb08f2acffbbfa21342 2024-11-23T16:16:14,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/.tmp/B/57cd3c9ba67c4bafb7fe46ed0631459c is 50, key is test_row_0/B:col10/1732378572131/Put/seqid=0 2024-11-23T16:16:14,247 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742357_1533 (size=12001) 2024-11-23T16:16:14,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(8855): Flush requested on 1930901e64343eeb48a7930647197c36 2024-11-23T16:16:14,280 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36. as already flushing 2024-11-23T16:16:14,301 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:16:14,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53788 deadline: 1732378634298, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:16:14,302 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:16:14,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53766 deadline: 1732378634298, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:16:14,302 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 1a7c5470b472,44715,1732378420375 for 'test_row_2' in 1930901e64343eeb48a7930647197c36 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2170 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-23T16:16:14,302 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 1a7c5470b472,44715,1732378420375 for 'test_row_2' in 1930901e64343eeb48a7930647197c36 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2169 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-23T16:16:14,302 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:14,303 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:14,303 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:14,303 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:16:14,303 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:14,303 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:16:14,304 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:16:14,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53810 deadline: 1732378634300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:16:14,305 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:14,305 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:14,305 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:16:14,306 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:16:14,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53830 deadline: 1732378634301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:16:14,306 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:16:14,307 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:14,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53774 deadline: 1732378634302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:16:14,307 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:14,307 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:16:14,307 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:14,307 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:14,307 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:16:14,410 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:16:14,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53810 deadline: 1732378634408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:16:14,410 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:14,410 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:14,410 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:16:14,421 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:16:14,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53830 deadline: 1732378634418, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:16:14,421 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:16:14,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53774 deadline: 1732378634418, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:16:14,421 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:14,422 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:14,422 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:16:14,422 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:14,422 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:14,422 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:16:14,620 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:16:14,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53810 deadline: 1732378634618, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:16:14,621 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:14,621 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:14,621 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:16:14,631 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:16:14,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53830 deadline: 1732378634628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:16:14,632 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:16:14,632 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:14,632 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:14,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53774 deadline: 1732378634628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:16:14,632 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:16:14,632 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:14,632 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:14,632 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:16:14,648 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=85 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/.tmp/B/57cd3c9ba67c4bafb7fe46ed0631459c 2024-11-23T16:16:14,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/.tmp/C/127137f654b54bad94ee1fcec6c22531 is 50, key is test_row_0/C:col10/1732378572131/Put/seqid=0 2024-11-23T16:16:14,657 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742358_1534 (size=12001) 2024-11-23T16:16:14,929 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:16:14,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53810 deadline: 1732378634927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:16:14,929 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:14,930 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:14,930 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:16:14,941 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:16:14,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53830 deadline: 1732378634938, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:16:14,941 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:16:14,941 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:14,941 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:14,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53774 deadline: 1732378634938, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:16:14,941 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:16:14,942 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:14,942 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:14,942 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:16:15,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-23T16:16:15,058 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=85 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/.tmp/C/127137f654b54bad94ee1fcec6c22531 2024-11-23T16:16:15,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/.tmp/A/7a7b39e3beb34eb08f2acffbbfa21342 as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/A/7a7b39e3beb34eb08f2acffbbfa21342 2024-11-23T16:16:15,065 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/A/7a7b39e3beb34eb08f2acffbbfa21342, entries=150, sequenceid=85, filesize=30.2 K 2024-11-23T16:16:15,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/.tmp/B/57cd3c9ba67c4bafb7fe46ed0631459c as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/B/57cd3c9ba67c4bafb7fe46ed0631459c 2024-11-23T16:16:15,070 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/B/57cd3c9ba67c4bafb7fe46ed0631459c, entries=150, sequenceid=85, filesize=11.7 K 2024-11-23T16:16:15,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/.tmp/C/127137f654b54bad94ee1fcec6c22531 as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/C/127137f654b54bad94ee1fcec6c22531 2024-11-23T16:16:15,074 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/C/127137f654b54bad94ee1fcec6c22531, entries=150, sequenceid=85, filesize=11.7 K 2024-11-23T16:16:15,075 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=80.51 KB/82440 for 1930901e64343eeb48a7930647197c36 in 1256ms, sequenceid=85, compaction requested=false 2024-11-23T16:16:15,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(2603): Flush status journal for 1930901e64343eeb48a7930647197c36: 2024-11-23T16:16:15,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36. 2024-11-23T16:16:15,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=143 2024-11-23T16:16:15,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45799 {}] master.HMaster(4169): Remote procedure done, pid=143 2024-11-23T16:16:15,078 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=143, resume processing ppid=142 2024-11-23T16:16:15,078 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=143, ppid=142, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1750 sec 2024-11-23T16:16:15,080 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=142, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=142, table=TestAcidGuarantees in 2.1800 sec 2024-11-23T16:16:15,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(8855): Flush requested on 1930901e64343eeb48a7930647197c36 2024-11-23T16:16:15,440 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1930901e64343eeb48a7930647197c36 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-23T16:16:15,440 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1930901e64343eeb48a7930647197c36, store=A 2024-11-23T16:16:15,440 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-23T16:16:15,440 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1930901e64343eeb48a7930647197c36, store=B 2024-11-23T16:16:15,440 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-23T16:16:15,441 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1930901e64343eeb48a7930647197c36, store=C 2024-11-23T16:16:15,441 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-23T16:16:15,447 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411231dbfaf115d994a3a97dc6ddcfb418b7c_1930901e64343eeb48a7930647197c36 is 50, key is test_row_0/A:col10/1732378574295/Put/seqid=0 2024-11-23T16:16:15,451 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742359_1535 (size=14594) 2024-11-23T16:16:15,474 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:16:15,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53830 deadline: 1732378635469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:16:15,474 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:15,474 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:15,474 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:16:15,474 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:16:15,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53774 deadline: 1732378635470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:16:15,475 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:15,475 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:15,475 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:16:15,475 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:16:15,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53810 deadline: 1732378635471, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:16:15,475 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:15,476 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:15,476 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:16:15,582 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:16:15,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53830 deadline: 1732378635578, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:16:15,582 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:16:15,582 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:15,582 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:15,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53774 deadline: 1732378635578, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:16:15,582 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:16:15,583 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:15,583 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:16:15,583 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:15,583 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:16:15,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53810 deadline: 1732378635578, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:16:15,583 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:15,583 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:15,583 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:16:15,790 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:16:15,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53830 deadline: 1732378635788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:16:15,790 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:16:15,790 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:15,790 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:15,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53774 deadline: 1732378635788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:16:15,790 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:16:15,791 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:16:15,791 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:15,791 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:15,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53810 deadline: 1732378635788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:16:15,791 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:16:15,791 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:15,791 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:15,791 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:16:15,852 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:15,855 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411231dbfaf115d994a3a97dc6ddcfb418b7c_1930901e64343eeb48a7930647197c36 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411231dbfaf115d994a3a97dc6ddcfb418b7c_1930901e64343eeb48a7930647197c36 2024-11-23T16:16:15,856 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/.tmp/A/6d967083128640b8a90cbd86180672b3, store: [table=TestAcidGuarantees family=A region=1930901e64343eeb48a7930647197c36] 2024-11-23T16:16:15,856 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/.tmp/A/6d967083128640b8a90cbd86180672b3 is 175, key is test_row_0/A:col10/1732378574295/Put/seqid=0 2024-11-23T16:16:15,866 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742360_1536 (size=39549) 2024-11-23T16:16:16,102 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:16:16,102 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:16:16,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53830 deadline: 1732378636097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:16:16,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53774 deadline: 1732378636097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:16:16,102 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:16:16,102 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:16,102 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:16,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53810 deadline: 1732378636098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:16:16,102 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:16,102 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:16,102 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:16:16,102 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:16:16,102 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:16,102 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:16,103 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:16:16,266 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=102, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/.tmp/A/6d967083128640b8a90cbd86180672b3 2024-11-23T16:16:16,274 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/.tmp/B/c16dc2b9be6a4d4c82455961056d43f8 is 50, key is test_row_0/B:col10/1732378574295/Put/seqid=0 2024-11-23T16:16:16,277 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742361_1537 (size=12001) 2024-11-23T16:16:16,322 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:16:16,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53788 deadline: 1732378636318, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:16:16,322 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:16:16,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53766 deadline: 1732378636318, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:16:16,322 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 1a7c5470b472,44715,1732378420375 for 'test_row_2' in 1930901e64343eeb48a7930647197c36 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4190 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-23T16:16:16,323 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:16,323 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:16,323 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:16:16,322 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 1a7c5470b472,44715,1732378420375 for 'test_row_2' in 1930901e64343eeb48a7930647197c36 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4189 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-23T16:16:16,323 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:16,323 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:16,323 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:16:16,611 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:16:16,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53830 deadline: 1732378636608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:16:16,612 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:16:16,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53774 deadline: 1732378636609, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:16:16,612 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:16,612 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:16,612 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:16:16,612 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:16:16,612 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:16,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53810 deadline: 1732378636609, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:16:16,612 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:16,612 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:16:16,613 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:16,613 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:16,613 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:16:16,678 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=102 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/.tmp/B/c16dc2b9be6a4d4c82455961056d43f8 2024-11-23T16:16:16,684 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/.tmp/C/cbca53d9e84b48a3ab7a5145274feb17 is 50, key is test_row_0/C:col10/1732378574295/Put/seqid=0 2024-11-23T16:16:16,687 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742362_1538 (size=12001) 2024-11-23T16:16:17,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-23T16:16:17,069 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-23T16:16:17,069 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45799 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-23T16:16:17,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45799 {}] procedure2.ProcedureExecutor(1139): Stored pid=144, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=144, table=TestAcidGuarantees 2024-11-23T16:16:17,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-23T16:16:17,071 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=144, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=144, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-23T16:16:17,071 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=144, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=144, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-23T16:16:17,072 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=145, ppid=144, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-23T16:16:17,088 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=102 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/.tmp/C/cbca53d9e84b48a3ab7a5145274feb17 2024-11-23T16:16:17,091 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/.tmp/A/6d967083128640b8a90cbd86180672b3 as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/A/6d967083128640b8a90cbd86180672b3 2024-11-23T16:16:17,095 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/A/6d967083128640b8a90cbd86180672b3, entries=200, sequenceid=102, filesize=38.6 K 2024-11-23T16:16:17,095 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/.tmp/B/c16dc2b9be6a4d4c82455961056d43f8 as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/B/c16dc2b9be6a4d4c82455961056d43f8 2024-11-23T16:16:17,098 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/B/c16dc2b9be6a4d4c82455961056d43f8, entries=150, sequenceid=102, filesize=11.7 K 2024-11-23T16:16:17,099 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/.tmp/C/cbca53d9e84b48a3ab7a5145274feb17 as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/C/cbca53d9e84b48a3ab7a5145274feb17 2024-11-23T16:16:17,102 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/C/cbca53d9e84b48a3ab7a5145274feb17, entries=150, sequenceid=102, filesize=11.7 K 2024-11-23T16:16:17,103 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=93.93 KB/96180 for 1930901e64343eeb48a7930647197c36 in 1663ms, sequenceid=102, compaction requested=true 2024-11-23T16:16:17,103 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1930901e64343eeb48a7930647197c36: 2024-11-23T16:16:17,103 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1930901e64343eeb48a7930647197c36:A, priority=-2147483648, current under compaction store size is 1 2024-11-23T16:16:17,103 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T16:16:17,103 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1930901e64343eeb48a7930647197c36:B, priority=-2147483648, current under compaction store size is 2 2024-11-23T16:16:17,103 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T16:16:17,103 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T16:16:17,103 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1930901e64343eeb48a7930647197c36:C, priority=-2147483648, current under compaction store size is 3 2024-11-23T16:16:17,103 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T16:16:17,103 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T16:16:17,105 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101562 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T16:16:17,105 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T16:16:17,105 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HStore(1541): 1930901e64343eeb48a7930647197c36/A is initiating minor compaction (all files) 2024-11-23T16:16:17,105 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HStore(1541): 1930901e64343eeb48a7930647197c36/B is initiating minor compaction (all files) 2024-11-23T16:16:17,105 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1930901e64343eeb48a7930647197c36/A in TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36. 2024-11-23T16:16:17,105 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1930901e64343eeb48a7930647197c36/B in TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36. 2024-11-23T16:16:17,105 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/B/1c152fa1818f41ada2915546d5afae8c, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/B/57cd3c9ba67c4bafb7fe46ed0631459c, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/B/c16dc2b9be6a4d4c82455961056d43f8] into tmpdir=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/.tmp, totalSize=35.3 K 2024-11-23T16:16:17,105 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/A/9032ef42731641baba6d8904f48110c4, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/A/7a7b39e3beb34eb08f2acffbbfa21342, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/A/6d967083128640b8a90cbd86180672b3] into tmpdir=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/.tmp, totalSize=99.2 K 2024-11-23T16:16:17,105 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36. 2024-11-23T16:16:17,105 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36. files: [hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/A/9032ef42731641baba6d8904f48110c4, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/A/7a7b39e3beb34eb08f2acffbbfa21342, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/A/6d967083128640b8a90cbd86180672b3] 2024-11-23T16:16:17,105 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.Compactor(225): Compacting 1c152fa1818f41ada2915546d5afae8c, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=65, earliestPutTs=1732378569950 2024-11-23T16:16:17,106 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9032ef42731641baba6d8904f48110c4, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=65, earliestPutTs=1732378569950 2024-11-23T16:16:17,106 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7a7b39e3beb34eb08f2acffbbfa21342, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=85, earliestPutTs=1732378572124 2024-11-23T16:16:17,106 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.Compactor(225): Compacting 57cd3c9ba67c4bafb7fe46ed0631459c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=85, earliestPutTs=1732378572124 2024-11-23T16:16:17,106 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.Compactor(225): Compacting c16dc2b9be6a4d4c82455961056d43f8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=102, earliestPutTs=1732378574295 2024-11-23T16:16:17,106 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6d967083128640b8a90cbd86180672b3, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=102, earliestPutTs=1732378574295 2024-11-23T16:16:17,111 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=1930901e64343eeb48a7930647197c36] 2024-11-23T16:16:17,114 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1930901e64343eeb48a7930647197c36#B#compaction#456 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-23T16:16:17,114 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/.tmp/B/e7717a4adce94acca5a666b8d8492954 is 50, key is test_row_0/B:col10/1732378574295/Put/seqid=0 2024-11-23T16:16:17,114 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411234697bab3f4f84362983746108367ef71_1930901e64343eeb48a7930647197c36 store=[table=TestAcidGuarantees family=A region=1930901e64343eeb48a7930647197c36] 2024-11-23T16:16:17,117 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411234697bab3f4f84362983746108367ef71_1930901e64343eeb48a7930647197c36, store=[table=TestAcidGuarantees family=A region=1930901e64343eeb48a7930647197c36] 2024-11-23T16:16:17,117 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411234697bab3f4f84362983746108367ef71_1930901e64343eeb48a7930647197c36 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=1930901e64343eeb48a7930647197c36] 2024-11-23T16:16:17,126 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742364_1540 (size=4469) 2024-11-23T16:16:17,130 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742363_1539 (size=12207) 2024-11-23T16:16:17,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-23T16:16:17,224 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-11-23T16:16:17,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36. 2024-11-23T16:16:17,224 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2902): Flushing 1930901e64343eeb48a7930647197c36 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-23T16:16:17,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1930901e64343eeb48a7930647197c36, store=A 2024-11-23T16:16:17,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-23T16:16:17,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1930901e64343eeb48a7930647197c36, store=B 2024-11-23T16:16:17,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-23T16:16:17,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1930901e64343eeb48a7930647197c36, store=C 2024-11-23T16:16:17,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-23T16:16:17,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112310273c7bb5714d088022d2f4768964bc_1930901e64343eeb48a7930647197c36 is 50, key is test_row_0/A:col10/1732378575468/Put/seqid=0 2024-11-23T16:16:17,238 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742365_1541 (size=12154) 2024-11-23T16:16:17,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-23T16:16:17,528 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1930901e64343eeb48a7930647197c36#A#compaction#455 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T16:16:17,529 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/.tmp/A/58f564b66ed54635a5e31adb9aeba85f is 175, key is test_row_0/A:col10/1732378574295/Put/seqid=0 2024-11-23T16:16:17,537 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742366_1542 (size=31161) 2024-11-23T16:16:17,538 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/.tmp/B/e7717a4adce94acca5a666b8d8492954 as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/B/e7717a4adce94acca5a666b8d8492954 2024-11-23T16:16:17,541 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/.tmp/A/58f564b66ed54635a5e31adb9aeba85f as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/A/58f564b66ed54635a5e31adb9aeba85f 2024-11-23T16:16:17,543 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1930901e64343eeb48a7930647197c36/B of 1930901e64343eeb48a7930647197c36 into e7717a4adce94acca5a666b8d8492954(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T16:16:17,543 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1930901e64343eeb48a7930647197c36: 2024-11-23T16:16:17,543 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., storeName=1930901e64343eeb48a7930647197c36/B, priority=13, startTime=1732378577103; duration=0sec 2024-11-23T16:16:17,543 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T16:16:17,543 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1930901e64343eeb48a7930647197c36:B 2024-11-23T16:16:17,543 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T16:16:17,544 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T16:16:17,544 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HStore(1541): 1930901e64343eeb48a7930647197c36/C is initiating minor compaction (all files) 2024-11-23T16:16:17,544 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1930901e64343eeb48a7930647197c36/C in TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36. 2024-11-23T16:16:17,544 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/C/18f634731ea24c58b1b2b3c86fcb1bf6, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/C/127137f654b54bad94ee1fcec6c22531, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/C/cbca53d9e84b48a3ab7a5145274feb17] into tmpdir=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/.tmp, totalSize=35.3 K 2024-11-23T16:16:17,545 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.Compactor(225): Compacting 18f634731ea24c58b1b2b3c86fcb1bf6, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=65, earliestPutTs=1732378569950 2024-11-23T16:16:17,545 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.Compactor(225): Compacting 127137f654b54bad94ee1fcec6c22531, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=85, earliestPutTs=1732378572124 2024-11-23T16:16:17,545 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.Compactor(225): Compacting cbca53d9e84b48a3ab7a5145274feb17, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=102, earliestPutTs=1732378574295 2024-11-23T16:16:17,546 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1930901e64343eeb48a7930647197c36/A of 1930901e64343eeb48a7930647197c36 into 58f564b66ed54635a5e31adb9aeba85f(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T16:16:17,546 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1930901e64343eeb48a7930647197c36: 2024-11-23T16:16:17,546 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., storeName=1930901e64343eeb48a7930647197c36/A, priority=13, startTime=1732378577103; duration=0sec 2024-11-23T16:16:17,546 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T16:16:17,546 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1930901e64343eeb48a7930647197c36:A 2024-11-23T16:16:17,551 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1930901e64343eeb48a7930647197c36#C#compaction#458 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T16:16:17,552 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/.tmp/C/83f7f479ed3c468899cbf3528bd681d2 is 50, key is test_row_0/C:col10/1732378574295/Put/seqid=0 2024-11-23T16:16:17,555 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742367_1543 (size=12207) 2024-11-23T16:16:17,559 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/.tmp/C/83f7f479ed3c468899cbf3528bd681d2 as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/C/83f7f479ed3c468899cbf3528bd681d2 2024-11-23T16:16:17,563 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1930901e64343eeb48a7930647197c36/C of 1930901e64343eeb48a7930647197c36 into 83f7f479ed3c468899cbf3528bd681d2(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T16:16:17,563 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1930901e64343eeb48a7930647197c36: 2024-11-23T16:16:17,563 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., storeName=1930901e64343eeb48a7930647197c36/C, priority=13, startTime=1732378577103; duration=0sec 2024-11-23T16:16:17,563 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T16:16:17,563 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1930901e64343eeb48a7930647197c36:C 2024-11-23T16:16:17,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(8855): Flush requested on 1930901e64343eeb48a7930647197c36 2024-11-23T16:16:17,620 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36. as already flushing 2024-11-23T16:16:17,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:17,642 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112310273c7bb5714d088022d2f4768964bc_1930901e64343eeb48a7930647197c36 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112310273c7bb5714d088022d2f4768964bc_1930901e64343eeb48a7930647197c36 2024-11-23T16:16:17,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/.tmp/A/3f5f1c6d7b4e43e28da90dcb2b10f617, store: [table=TestAcidGuarantees family=A region=1930901e64343eeb48a7930647197c36] 2024-11-23T16:16:17,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/.tmp/A/3f5f1c6d7b4e43e28da90dcb2b10f617 is 175, key is test_row_0/A:col10/1732378575468/Put/seqid=0 2024-11-23T16:16:17,647 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742368_1544 (size=30955) 2024-11-23T16:16:17,648 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:16:17,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53810 deadline: 1732378637644, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:16:17,648 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:17,648 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:17,648 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:16:17,649 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:16:17,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53774 deadline: 1732378637645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:16:17,650 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:16:17,650 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:17,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53830 deadline: 1732378637646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:16:17,650 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:17,650 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:16:17,650 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:17,650 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:17,650 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:16:17,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-23T16:16:17,761 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:16:17,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53810 deadline: 1732378637758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:16:17,762 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:16:17,762 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:17,762 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:17,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53774 deadline: 1732378637758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:16:17,762 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:16:17,762 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:17,762 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:16:17,762 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:17,762 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:16:17,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53830 deadline: 1732378637758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:16:17,763 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:17,763 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:17,763 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:16:17,971 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:16:17,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53810 deadline: 1732378637968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:16:17,972 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:17,972 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:16:17,972 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:17,972 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:16:17,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53774 deadline: 1732378637968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:16:17,972 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:17,972 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:17,972 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:16:17,972 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:16:17,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53830 deadline: 1732378637968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:16:17,972 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:17,973 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:17,973 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:16:18,048 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=119, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/.tmp/A/3f5f1c6d7b4e43e28da90dcb2b10f617 2024-11-23T16:16:18,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/.tmp/B/ffdeaa2c96f34aedbf00684d36a3bb34 is 50, key is test_row_0/B:col10/1732378575468/Put/seqid=0 2024-11-23T16:16:18,058 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742369_1545 (size=12001) 2024-11-23T16:16:18,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-23T16:16:18,281 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:16:18,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53810 deadline: 1732378638278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:16:18,281 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:16:18,281 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:18,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53774 deadline: 1732378638278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:16:18,281 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:18,281 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:16:18,281 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:18,281 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:16:18,281 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:18,282 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:16:18,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53830 deadline: 1732378638278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:16:18,282 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:18,282 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:18,282 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:16:18,458 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=119 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/.tmp/B/ffdeaa2c96f34aedbf00684d36a3bb34 2024-11-23T16:16:18,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/.tmp/C/302a6aeb45de4b8f8365f3206713d10c is 50, key is test_row_0/C:col10/1732378575468/Put/seqid=0 2024-11-23T16:16:18,469 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742370_1546 (size=12001) 2024-11-23T16:16:18,469 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=119 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/.tmp/C/302a6aeb45de4b8f8365f3206713d10c 2024-11-23T16:16:18,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/.tmp/A/3f5f1c6d7b4e43e28da90dcb2b10f617 as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/A/3f5f1c6d7b4e43e28da90dcb2b10f617 2024-11-23T16:16:18,475 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/A/3f5f1c6d7b4e43e28da90dcb2b10f617, entries=150, sequenceid=119, filesize=30.2 K 2024-11-23T16:16:18,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/.tmp/B/ffdeaa2c96f34aedbf00684d36a3bb34 as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/B/ffdeaa2c96f34aedbf00684d36a3bb34 2024-11-23T16:16:18,479 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/B/ffdeaa2c96f34aedbf00684d36a3bb34, entries=150, sequenceid=119, filesize=11.7 K 2024-11-23T16:16:18,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/.tmp/C/302a6aeb45de4b8f8365f3206713d10c as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/C/302a6aeb45de4b8f8365f3206713d10c 2024-11-23T16:16:18,482 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/C/302a6aeb45de4b8f8365f3206713d10c, entries=150, sequenceid=119, filesize=11.7 K 2024-11-23T16:16:18,483 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=87.22 KB/89310 for 1930901e64343eeb48a7930647197c36 in 1259ms, sequenceid=119, compaction requested=false 2024-11-23T16:16:18,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2603): Flush status journal for 1930901e64343eeb48a7930647197c36: 2024-11-23T16:16:18,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36. 2024-11-23T16:16:18,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=145 2024-11-23T16:16:18,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45799 {}] master.HMaster(4169): Remote procedure done, pid=145 2024-11-23T16:16:18,486 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=145, resume processing ppid=144 2024-11-23T16:16:18,486 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=145, ppid=144, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4120 sec 2024-11-23T16:16:18,488 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=144, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=144, table=TestAcidGuarantees in 1.4180 sec 2024-11-23T16:16:18,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(8855): Flush requested on 1930901e64343eeb48a7930647197c36 2024-11-23T16:16:18,791 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1930901e64343eeb48a7930647197c36 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-23T16:16:18,792 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1930901e64343eeb48a7930647197c36, store=A 2024-11-23T16:16:18,792 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-23T16:16:18,792 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1930901e64343eeb48a7930647197c36, store=B 2024-11-23T16:16:18,792 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-23T16:16:18,792 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1930901e64343eeb48a7930647197c36, store=C 2024-11-23T16:16:18,792 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-23T16:16:18,798 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411235ff4927dbec8486c94f4017980f1a48f_1930901e64343eeb48a7930647197c36 is 50, key is test_row_0/A:col10/1732378578791/Put/seqid=0 2024-11-23T16:16:18,810 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742371_1547 (size=12304) 2024-11-23T16:16:18,812 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:16:18,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53774 deadline: 1732378638807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:16:18,813 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:18,813 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:16:18,813 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:18,813 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:16:18,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53830 deadline: 1732378638808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:16:18,813 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:18,813 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:18,813 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:16:18,816 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:16:18,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53810 deadline: 1732378638813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:16:18,816 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:18,817 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:18,817 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:16:18,922 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:16:18,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53774 deadline: 1732378638918, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:16:18,922 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:16:18,922 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:18,922 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:18,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53830 deadline: 1732378638918, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:16:18,922 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:16:18,922 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:16:18,922 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:18,922 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:18,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53810 deadline: 1732378638918, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:16:18,923 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:16:18,923 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:18,923 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:18,923 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:16:19,131 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:16:19,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53774 deadline: 1732378639128, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:16:19,132 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:16:19,132 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:19,132 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:19,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53830 deadline: 1732378639128, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:16:19,132 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:16:19,132 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:16:19,132 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:19,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53810 deadline: 1732378639129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:16:19,132 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:19,132 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:16:19,132 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:19,133 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:19,133 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:16:19,210 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:19,214 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411235ff4927dbec8486c94f4017980f1a48f_1930901e64343eeb48a7930647197c36 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411235ff4927dbec8486c94f4017980f1a48f_1930901e64343eeb48a7930647197c36 2024-11-23T16:16:19,214 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/.tmp/A/fcea5c7418154dffac3ea561659c88e7, store: [table=TestAcidGuarantees family=A region=1930901e64343eeb48a7930647197c36] 2024-11-23T16:16:19,215 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/.tmp/A/fcea5c7418154dffac3ea561659c88e7 is 175, key is test_row_0/A:col10/1732378578791/Put/seqid=0 2024-11-23T16:16:19,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-23T16:16:19,219 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-23T16:16:19,219 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45799 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-23T16:16:19,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45799 {}] procedure2.ProcedureExecutor(1139): Stored pid=146, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees 2024-11-23T16:16:19,221 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=146, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-23T16:16:19,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-23T16:16:19,222 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=146, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-23T16:16:19,222 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=147, ppid=146, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-23T16:16:19,234 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742372_1548 (size=31105) 2024-11-23T16:16:19,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-23T16:16:19,374 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-23T16:16:19,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36. 2024-11-23T16:16:19,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36. as already flushing 2024-11-23T16:16:19,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36. 2024-11-23T16:16:19,374 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => 1930901e64343eeb48a7930647197c36, NAME => 'TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:16:19,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => 1930901e64343eeb48a7930647197c36, NAME => 'TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:16:19,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45799 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1930901e64343eeb48a7930647197c36, NAME => 'TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1930901e64343eeb48a7930647197c36, NAME => 'TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:16:19,442 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:16:19,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53774 deadline: 1732378639438, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:16:19,442 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:16:19,442 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:19,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53830 deadline: 1732378639438, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:16:19,442 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:19,442 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:16:19,443 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:16:19,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53810 deadline: 1732378639438, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:16:19,443 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:19,443 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:19,443 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:16:19,443 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:19,443 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:19,443 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:16:19,527 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-23T16:16:19,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36. 2024-11-23T16:16:19,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36. as already flushing 2024-11-23T16:16:19,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36. 2024-11-23T16:16:19,528 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => 1930901e64343eeb48a7930647197c36, NAME => 'TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:16:19,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => 1930901e64343eeb48a7930647197c36, NAME => 'TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:16:19,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45799 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1930901e64343eeb48a7930647197c36, NAME => 'TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1930901e64343eeb48a7930647197c36, NAME => 'TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:16:19,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-23T16:16:19,634 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=140, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/.tmp/A/fcea5c7418154dffac3ea561659c88e7 2024-11-23T16:16:19,642 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/.tmp/B/96f7bdc38ca3425995ffeafab1347dff is 50, key is test_row_0/B:col10/1732378578791/Put/seqid=0 2024-11-23T16:16:19,646 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742373_1549 (size=12151) 2024-11-23T16:16:19,680 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-23T16:16:19,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36. 2024-11-23T16:16:19,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36. as already flushing 2024-11-23T16:16:19,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36. 2024-11-23T16:16:19,680 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => 1930901e64343eeb48a7930647197c36, NAME => 'TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:16:19,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => 1930901e64343eeb48a7930647197c36, NAME => 'TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:16:19,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45799 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1930901e64343eeb48a7930647197c36, NAME => 'TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1930901e64343eeb48a7930647197c36, NAME => 'TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:16:19,833 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-23T16:16:19,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36. 2024-11-23T16:16:19,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36. as already flushing 2024-11-23T16:16:19,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36. 2024-11-23T16:16:19,834 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => 1930901e64343eeb48a7930647197c36, NAME => 'TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:16:19,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => 1930901e64343eeb48a7930647197c36, NAME => 'TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:16:19,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45799 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1930901e64343eeb48a7930647197c36, NAME => 'TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1930901e64343eeb48a7930647197c36, NAME => 'TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:16:19,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-23T16:16:19,951 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:16:19,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53774 deadline: 1732378639947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:16:19,951 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:16:19,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53830 deadline: 1732378639948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:16:19,951 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:19,951 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:19,951 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:16:19,951 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:16:19,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53810 deadline: 1732378639948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:16:19,951 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:19,951 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:19,951 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:16:19,951 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:19,952 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:19,952 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:16:19,986 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-23T16:16:19,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36. 2024-11-23T16:16:19,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36. as already flushing 2024-11-23T16:16:19,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36. 2024-11-23T16:16:19,986 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => 1930901e64343eeb48a7930647197c36, NAME => 'TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:16:19,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => 1930901e64343eeb48a7930647197c36, NAME => 'TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:16:19,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45799 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1930901e64343eeb48a7930647197c36, NAME => 'TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1930901e64343eeb48a7930647197c36, NAME => 'TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:16:20,047 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=140 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/.tmp/B/96f7bdc38ca3425995ffeafab1347dff 2024-11-23T16:16:20,054 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/.tmp/C/7e9f3e3dd9354c3da45c652b735bacaf is 50, key is test_row_0/C:col10/1732378578791/Put/seqid=0 2024-11-23T16:16:20,057 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742374_1550 (size=12151) 2024-11-23T16:16:20,139 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-23T16:16:20,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36. 2024-11-23T16:16:20,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36. as already flushing 2024-11-23T16:16:20,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36. 2024-11-23T16:16:20,139 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => 1930901e64343eeb48a7930647197c36, NAME => 'TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:16:20,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => 1930901e64343eeb48a7930647197c36, NAME => 'TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:16:20,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45799 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1930901e64343eeb48a7930647197c36, NAME => 'TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1930901e64343eeb48a7930647197c36, NAME => 'TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:16:20,292 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-23T16:16:20,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36. 2024-11-23T16:16:20,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36. as already flushing 2024-11-23T16:16:20,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36. 2024-11-23T16:16:20,293 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => 1930901e64343eeb48a7930647197c36, NAME => 'TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:16:20,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => 1930901e64343eeb48a7930647197c36, NAME => 'TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:16:20,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45799 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1930901e64343eeb48a7930647197c36, NAME => 'TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1930901e64343eeb48a7930647197c36, NAME => 'TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:16:20,328 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:16:20,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53766 deadline: 1732378640327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:16:20,328 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 1a7c5470b472,44715,1732378420375 for 'test_row_2' in 1930901e64343eeb48a7930647197c36 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8195 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-23T16:16:20,329 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:20,329 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:20,329 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:16:20,350 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:16:20,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53788 deadline: 1732378640348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:16:20,351 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 1a7c5470b472,44715,1732378420375 for 'test_row_2' in 1930901e64343eeb48a7930647197c36 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8218 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-23T16:16:20,351 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:20,351 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:20,351 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:16:20,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-23T16:16:20,446 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-23T16:16:20,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36. 2024-11-23T16:16:20,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36. as already flushing 2024-11-23T16:16:20,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36. 2024-11-23T16:16:20,446 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => 1930901e64343eeb48a7930647197c36, NAME => 'TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:16:20,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => 1930901e64343eeb48a7930647197c36, NAME => 'TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:16:20,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45799 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1930901e64343eeb48a7930647197c36, NAME => 'TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1930901e64343eeb48a7930647197c36, NAME => 'TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:16:20,457 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=140 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/.tmp/C/7e9f3e3dd9354c3da45c652b735bacaf 2024-11-23T16:16:20,461 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/.tmp/A/fcea5c7418154dffac3ea561659c88e7 as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/A/fcea5c7418154dffac3ea561659c88e7 2024-11-23T16:16:20,464 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/A/fcea5c7418154dffac3ea561659c88e7, entries=150, sequenceid=140, filesize=30.4 K 2024-11-23T16:16:20,465 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/.tmp/B/96f7bdc38ca3425995ffeafab1347dff as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/B/96f7bdc38ca3425995ffeafab1347dff 2024-11-23T16:16:20,467 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/B/96f7bdc38ca3425995ffeafab1347dff, entries=150, sequenceid=140, filesize=11.9 K 2024-11-23T16:16:20,468 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/.tmp/C/7e9f3e3dd9354c3da45c652b735bacaf as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/C/7e9f3e3dd9354c3da45c652b735bacaf 2024-11-23T16:16:20,471 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/C/7e9f3e3dd9354c3da45c652b735bacaf, entries=150, sequenceid=140, filesize=11.9 K 2024-11-23T16:16:20,472 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 1930901e64343eeb48a7930647197c36 in 1681ms, sequenceid=140, compaction requested=true 2024-11-23T16:16:20,472 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1930901e64343eeb48a7930647197c36: 2024-11-23T16:16:20,472 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1930901e64343eeb48a7930647197c36:A, priority=-2147483648, current under compaction store size is 1 2024-11-23T16:16:20,472 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T16:16:20,472 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1930901e64343eeb48a7930647197c36:B, priority=-2147483648, current under compaction store size is 2 2024-11-23T16:16:20,472 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T16:16:20,472 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1930901e64343eeb48a7930647197c36:C, priority=-2147483648, current under compaction store size is 3 2024-11-23T16:16:20,472 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T16:16:20,472 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T16:16:20,472 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T16:16:20,473 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36359 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T16:16:20,473 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93221 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T16:16:20,473 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HStore(1541): 1930901e64343eeb48a7930647197c36/B is initiating minor compaction (all files) 2024-11-23T16:16:20,473 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HStore(1541): 1930901e64343eeb48a7930647197c36/A is initiating minor compaction (all files) 2024-11-23T16:16:20,473 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1930901e64343eeb48a7930647197c36/B in TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36. 2024-11-23T16:16:20,473 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1930901e64343eeb48a7930647197c36/A in TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36. 2024-11-23T16:16:20,473 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/B/e7717a4adce94acca5a666b8d8492954, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/B/ffdeaa2c96f34aedbf00684d36a3bb34, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/B/96f7bdc38ca3425995ffeafab1347dff] into tmpdir=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/.tmp, totalSize=35.5 K 2024-11-23T16:16:20,473 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/A/58f564b66ed54635a5e31adb9aeba85f, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/A/3f5f1c6d7b4e43e28da90dcb2b10f617, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/A/fcea5c7418154dffac3ea561659c88e7] into tmpdir=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/.tmp, totalSize=91.0 K 2024-11-23T16:16:20,473 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36. 2024-11-23T16:16:20,473 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36. files: [hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/A/58f564b66ed54635a5e31adb9aeba85f, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/A/3f5f1c6d7b4e43e28da90dcb2b10f617, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/A/fcea5c7418154dffac3ea561659c88e7] 2024-11-23T16:16:20,474 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.Compactor(225): Compacting e7717a4adce94acca5a666b8d8492954, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=102, earliestPutTs=1732378574295 2024-11-23T16:16:20,474 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.Compactor(225): Compacting 58f564b66ed54635a5e31adb9aeba85f, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=102, earliestPutTs=1732378574295 2024-11-23T16:16:20,474 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.Compactor(225): Compacting ffdeaa2c96f34aedbf00684d36a3bb34, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1732378575464 2024-11-23T16:16:20,474 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.Compactor(225): Compacting 96f7bdc38ca3425995ffeafab1347dff, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=140, earliestPutTs=1732378577642 2024-11-23T16:16:20,474 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3f5f1c6d7b4e43e28da90dcb2b10f617, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1732378575464 2024-11-23T16:16:20,476 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.Compactor(225): Compacting fcea5c7418154dffac3ea561659c88e7, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=140, earliestPutTs=1732378577642 2024-11-23T16:16:20,481 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=1930901e64343eeb48a7930647197c36] 2024-11-23T16:16:20,483 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1930901e64343eeb48a7930647197c36#B#compaction#464 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-23T16:16:20,483 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/.tmp/B/79af371c8b3343d09fc762b6c3b3b392 is 50, key is test_row_0/B:col10/1732378578791/Put/seqid=0 2024-11-23T16:16:20,484 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241123983e59b937ab476daa4b5c1a707b241d_1930901e64343eeb48a7930647197c36 store=[table=TestAcidGuarantees family=A region=1930901e64343eeb48a7930647197c36] 2024-11-23T16:16:20,486 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241123983e59b937ab476daa4b5c1a707b241d_1930901e64343eeb48a7930647197c36, store=[table=TestAcidGuarantees family=A region=1930901e64343eeb48a7930647197c36] 2024-11-23T16:16:20,486 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241123983e59b937ab476daa4b5c1a707b241d_1930901e64343eeb48a7930647197c36 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=1930901e64343eeb48a7930647197c36] 2024-11-23T16:16:20,488 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742375_1551 (size=12459) 2024-11-23T16:16:20,493 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/.tmp/B/79af371c8b3343d09fc762b6c3b3b392 as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/B/79af371c8b3343d09fc762b6c3b3b392 2024-11-23T16:16:20,498 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1930901e64343eeb48a7930647197c36/B of 1930901e64343eeb48a7930647197c36 into 79af371c8b3343d09fc762b6c3b3b392(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T16:16:20,498 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1930901e64343eeb48a7930647197c36: 2024-11-23T16:16:20,498 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., storeName=1930901e64343eeb48a7930647197c36/B, priority=13, startTime=1732378580472; duration=0sec 2024-11-23T16:16:20,498 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T16:16:20,498 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1930901e64343eeb48a7930647197c36:B 2024-11-23T16:16:20,498 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T16:16:20,499 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36359 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T16:16:20,499 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HStore(1541): 1930901e64343eeb48a7930647197c36/C is initiating minor compaction (all files) 2024-11-23T16:16:20,500 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1930901e64343eeb48a7930647197c36/C in TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36. 2024-11-23T16:16:20,500 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/C/83f7f479ed3c468899cbf3528bd681d2, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/C/302a6aeb45de4b8f8365f3206713d10c, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/C/7e9f3e3dd9354c3da45c652b735bacaf] into tmpdir=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/.tmp, totalSize=35.5 K 2024-11-23T16:16:20,500 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.Compactor(225): Compacting 83f7f479ed3c468899cbf3528bd681d2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=102, earliestPutTs=1732378574295 2024-11-23T16:16:20,500 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.Compactor(225): Compacting 302a6aeb45de4b8f8365f3206713d10c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1732378575464 2024-11-23T16:16:20,501 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.Compactor(225): Compacting 7e9f3e3dd9354c3da45c652b735bacaf, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=140, earliestPutTs=1732378577642 2024-11-23T16:16:20,510 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1930901e64343eeb48a7930647197c36#C#compaction#466 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-23T16:16:20,510 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/.tmp/C/891b5a1cb3fb4bab82596b5bfe06ccfb is 50, key is test_row_0/C:col10/1732378578791/Put/seqid=0 2024-11-23T16:16:20,512 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742376_1552 (size=4469) 2024-11-23T16:16:20,520 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742377_1553 (size=12459) 2024-11-23T16:16:20,524 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/.tmp/C/891b5a1cb3fb4bab82596b5bfe06ccfb as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/C/891b5a1cb3fb4bab82596b5bfe06ccfb 2024-11-23T16:16:20,528 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1930901e64343eeb48a7930647197c36/C of 1930901e64343eeb48a7930647197c36 into 891b5a1cb3fb4bab82596b5bfe06ccfb(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T16:16:20,528 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1930901e64343eeb48a7930647197c36: 2024-11-23T16:16:20,528 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., storeName=1930901e64343eeb48a7930647197c36/C, priority=13, startTime=1732378580472; duration=0sec 2024-11-23T16:16:20,528 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T16:16:20,528 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1930901e64343eeb48a7930647197c36:C 2024-11-23T16:16:20,599 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-23T16:16:20,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36. 2024-11-23T16:16:20,599 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2902): Flushing 1930901e64343eeb48a7930647197c36 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-23T16:16:20,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1930901e64343eeb48a7930647197c36, store=A 2024-11-23T16:16:20,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-23T16:16:20,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1930901e64343eeb48a7930647197c36, store=B 2024-11-23T16:16:20,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-23T16:16:20,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1930901e64343eeb48a7930647197c36, store=C 2024-11-23T16:16:20,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-23T16:16:20,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241123ded1c4ea8aaa44dd9db45222062d413f_1930901e64343eeb48a7930647197c36 is 50, key is test_row_0/A:col10/1732378578806/Put/seqid=0 2024-11-23T16:16:20,610 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742378_1554 (size=12304) 2024-11-23T16:16:20,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:20,614 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241123ded1c4ea8aaa44dd9db45222062d413f_1930901e64343eeb48a7930647197c36 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123ded1c4ea8aaa44dd9db45222062d413f_1930901e64343eeb48a7930647197c36 2024-11-23T16:16:20,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/.tmp/A/88d5b8fb2b434b08a7306e7fb591eda5, store: [table=TestAcidGuarantees family=A region=1930901e64343eeb48a7930647197c36] 2024-11-23T16:16:20,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/.tmp/A/88d5b8fb2b434b08a7306e7fb591eda5 is 175, key is test_row_0/A:col10/1732378578806/Put/seqid=0 2024-11-23T16:16:20,618 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742379_1555 (size=31105) 2024-11-23T16:16:20,913 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1930901e64343eeb48a7930647197c36#A#compaction#465 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T16:16:20,913 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/.tmp/A/f7c5657eeafc4f5197739834d768e9ba is 175, key is test_row_0/A:col10/1732378578791/Put/seqid=0 2024-11-23T16:16:20,917 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742380_1556 (size=31413) 2024-11-23T16:16:20,921 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/.tmp/A/f7c5657eeafc4f5197739834d768e9ba as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/A/f7c5657eeafc4f5197739834d768e9ba 2024-11-23T16:16:20,926 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1930901e64343eeb48a7930647197c36/A of 1930901e64343eeb48a7930647197c36 into f7c5657eeafc4f5197739834d768e9ba(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T16:16:20,926 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1930901e64343eeb48a7930647197c36: 2024-11-23T16:16:20,926 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., storeName=1930901e64343eeb48a7930647197c36/A, priority=13, startTime=1732378580472; duration=0sec 2024-11-23T16:16:20,926 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T16:16:20,926 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1930901e64343eeb48a7930647197c36:A 2024-11-23T16:16:20,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(8855): Flush requested on 1930901e64343eeb48a7930647197c36 2024-11-23T16:16:20,960 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36. as already flushing 2024-11-23T16:16:20,986 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:16:20,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53774 deadline: 1732378640981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:16:20,987 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:16:20,987 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:20,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53810 deadline: 1732378640982, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:16:20,987 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:20,987 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:16:20,987 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:20,987 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:20,987 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:16:20,988 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:16:20,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53830 deadline: 1732378640984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:16:20,988 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:20,988 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:20,988 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:16:21,018 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=156, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/.tmp/A/88d5b8fb2b434b08a7306e7fb591eda5 2024-11-23T16:16:21,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/.tmp/B/dda8b67939ed4d189432c59113213813 is 50, key is test_row_0/B:col10/1732378578806/Put/seqid=0 2024-11-23T16:16:21,028 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742381_1557 (size=12151) 2024-11-23T16:16:21,101 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:16:21,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53774 deadline: 1732378641098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:16:21,102 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:16:21,102 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:21,102 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:21,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53810 deadline: 1732378641098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:16:21,102 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:16:21,102 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:16:21,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53830 deadline: 1732378641098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:16:21,102 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:21,102 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:21,102 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:16:21,102 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:21,102 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:21,102 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:16:21,311 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:16:21,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53810 deadline: 1732378641308, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:16:21,311 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:16:21,311 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:21,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53830 deadline: 1732378641308, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:16:21,311 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:21,311 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:16:21,312 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:21,312 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:21,312 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:16:21,312 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:16:21,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53774 deadline: 1732378641310, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:16:21,313 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:21,313 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:21,313 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:16:21,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-23T16:16:21,428 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/.tmp/B/dda8b67939ed4d189432c59113213813 2024-11-23T16:16:21,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/.tmp/C/654750eb4f1b4b5494ab2e6869513f84 is 50, key is test_row_0/C:col10/1732378578806/Put/seqid=0 2024-11-23T16:16:21,441 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742382_1558 (size=12151) 2024-11-23T16:16:21,442 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/.tmp/C/654750eb4f1b4b5494ab2e6869513f84 2024-11-23T16:16:21,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/.tmp/A/88d5b8fb2b434b08a7306e7fb591eda5 as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/A/88d5b8fb2b434b08a7306e7fb591eda5 2024-11-23T16:16:21,449 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/A/88d5b8fb2b434b08a7306e7fb591eda5, entries=150, sequenceid=156, filesize=30.4 K 2024-11-23T16:16:21,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/.tmp/B/dda8b67939ed4d189432c59113213813 as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/B/dda8b67939ed4d189432c59113213813 2024-11-23T16:16:21,453 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/B/dda8b67939ed4d189432c59113213813, entries=150, sequenceid=156, filesize=11.9 K 2024-11-23T16:16:21,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/.tmp/C/654750eb4f1b4b5494ab2e6869513f84 as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/C/654750eb4f1b4b5494ab2e6869513f84 2024-11-23T16:16:21,456 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/C/654750eb4f1b4b5494ab2e6869513f84, entries=150, sequenceid=156, filesize=11.9 K 2024-11-23T16:16:21,457 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 1930901e64343eeb48a7930647197c36 in 858ms, sequenceid=156, compaction requested=false 2024-11-23T16:16:21,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2603): Flush status journal for 1930901e64343eeb48a7930647197c36: 2024-11-23T16:16:21,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36. 2024-11-23T16:16:21,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=147 2024-11-23T16:16:21,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45799 {}] master.HMaster(4169): Remote procedure done, pid=147 2024-11-23T16:16:21,461 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=147, resume processing ppid=146 2024-11-23T16:16:21,461 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=147, ppid=146, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2370 sec 2024-11-23T16:16:21,464 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=146, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees in 2.2430 sec 2024-11-23T16:16:21,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(8855): Flush requested on 1930901e64343eeb48a7930647197c36 2024-11-23T16:16:21,620 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1930901e64343eeb48a7930647197c36 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-23T16:16:21,620 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1930901e64343eeb48a7930647197c36, store=A 2024-11-23T16:16:21,620 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-23T16:16:21,620 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1930901e64343eeb48a7930647197c36, store=B 2024-11-23T16:16:21,620 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-23T16:16:21,620 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1930901e64343eeb48a7930647197c36, store=C 2024-11-23T16:16:21,620 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-23T16:16:21,643 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411237c41755203e3479d89f7010f7d0895fa_1930901e64343eeb48a7930647197c36 is 50, key is test_row_0/A:col10/1732378581618/Put/seqid=0 2024-11-23T16:16:21,646 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742383_1559 (size=12304) 2024-11-23T16:16:21,665 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:16:21,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53774 deadline: 1732378641661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:16:21,666 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:21,666 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:21,666 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:16:21,666 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:16:21,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53810 deadline: 1732378641663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:16:21,666 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:21,666 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:21,666 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:16:21,668 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:16:21,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53830 deadline: 1732378641664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:16:21,668 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:21,668 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:21,668 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:16:21,769 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:16:21,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53774 deadline: 1732378641767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:16:21,770 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:21,770 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:21,770 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:16:21,780 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:16:21,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53810 deadline: 1732378641777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:16:21,781 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:16:21,781 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:21,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53830 deadline: 1732378641778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:16:21,781 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:21,781 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:16:21,781 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:21,781 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:21,781 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:16:21,980 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:16:21,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53774 deadline: 1732378641978, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:16:21,981 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:21,981 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:21,981 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:16:21,992 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:16:21,992 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:16:21,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53810 deadline: 1732378641987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:16:21,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53830 deadline: 1732378641988, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:16:21,992 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:21,992 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:21,992 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:21,992 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:16:21,992 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:21,992 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:16:22,046 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:22,050 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411237c41755203e3479d89f7010f7d0895fa_1930901e64343eeb48a7930647197c36 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411237c41755203e3479d89f7010f7d0895fa_1930901e64343eeb48a7930647197c36 2024-11-23T16:16:22,050 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/.tmp/A/46b30f27a7ed401da4d65c59ae93ef22, store: [table=TestAcidGuarantees family=A region=1930901e64343eeb48a7930647197c36] 2024-11-23T16:16:22,051 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/.tmp/A/46b30f27a7ed401da4d65c59ae93ef22 is 175, key is test_row_0/A:col10/1732378581618/Put/seqid=0 2024-11-23T16:16:22,054 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742384_1560 (size=31105) 2024-11-23T16:16:22,289 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:16:22,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53774 deadline: 1732378642288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:16:22,290 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:22,290 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:22,290 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:16:22,300 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:16:22,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53810 deadline: 1732378642298, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:16:22,301 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:22,301 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:22,301 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:16:22,301 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:16:22,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53830 deadline: 1732378642298, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:16:22,302 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:22,302 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:22,302 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:16:22,455 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=176, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/.tmp/A/46b30f27a7ed401da4d65c59ae93ef22 2024-11-23T16:16:22,461 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/.tmp/B/6a7a82c2177441b3bd67e96df594a850 is 50, key is test_row_0/B:col10/1732378581618/Put/seqid=0 2024-11-23T16:16:22,464 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742385_1561 (size=12151) 2024-11-23T16:16:22,800 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:16:22,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53774 deadline: 1732378642798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:16:22,800 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:22,801 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:22,801 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:16:22,812 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:16:22,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53810 deadline: 1732378642808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:16:22,812 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:16:22,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53830 deadline: 1732378642808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:16:22,812 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:22,812 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:22,812 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:16:22,813 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:22,813 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:22,813 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:16:22,865 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=176 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/.tmp/B/6a7a82c2177441b3bd67e96df594a850 2024-11-23T16:16:22,872 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/.tmp/C/68d8914b3afd4f6fbbfbb65d84a1628f is 50, key is test_row_0/C:col10/1732378581618/Put/seqid=0 2024-11-23T16:16:22,875 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742386_1562 (size=12151) 2024-11-23T16:16:22,876 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=176 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/.tmp/C/68d8914b3afd4f6fbbfbb65d84a1628f 2024-11-23T16:16:22,879 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/.tmp/A/46b30f27a7ed401da4d65c59ae93ef22 as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/A/46b30f27a7ed401da4d65c59ae93ef22 2024-11-23T16:16:22,883 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/A/46b30f27a7ed401da4d65c59ae93ef22, entries=150, sequenceid=176, filesize=30.4 K 2024-11-23T16:16:22,883 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/.tmp/B/6a7a82c2177441b3bd67e96df594a850 as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/B/6a7a82c2177441b3bd67e96df594a850 2024-11-23T16:16:22,886 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/B/6a7a82c2177441b3bd67e96df594a850, entries=150, sequenceid=176, filesize=11.9 K 2024-11-23T16:16:22,887 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/.tmp/C/68d8914b3afd4f6fbbfbb65d84a1628f as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/C/68d8914b3afd4f6fbbfbb65d84a1628f 2024-11-23T16:16:22,890 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/C/68d8914b3afd4f6fbbfbb65d84a1628f, entries=150, sequenceid=176, filesize=11.9 K 2024-11-23T16:16:22,891 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 1930901e64343eeb48a7930647197c36 in 1271ms, sequenceid=176, compaction requested=true 2024-11-23T16:16:22,891 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1930901e64343eeb48a7930647197c36: 2024-11-23T16:16:22,891 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1930901e64343eeb48a7930647197c36:A, priority=-2147483648, current under compaction store size is 1 2024-11-23T16:16:22,891 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T16:16:22,891 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1930901e64343eeb48a7930647197c36:B, priority=-2147483648, current under compaction store size is 2 2024-11-23T16:16:22,891 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T16:16:22,891 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1930901e64343eeb48a7930647197c36:C, priority=-2147483648, current under compaction store size is 3 2024-11-23T16:16:22,891 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T16:16:22,891 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T16:16:22,891 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T16:16:22,892 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T16:16:22,892 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93623 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T16:16:22,892 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HStore(1541): 1930901e64343eeb48a7930647197c36/A is initiating minor compaction (all files) 2024-11-23T16:16:22,892 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HStore(1541): 1930901e64343eeb48a7930647197c36/B is initiating minor compaction (all files) 2024-11-23T16:16:22,892 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1930901e64343eeb48a7930647197c36/A in TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36. 2024-11-23T16:16:22,892 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1930901e64343eeb48a7930647197c36/B in TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36. 2024-11-23T16:16:22,892 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/B/79af371c8b3343d09fc762b6c3b3b392, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/B/dda8b67939ed4d189432c59113213813, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/B/6a7a82c2177441b3bd67e96df594a850] into tmpdir=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/.tmp, totalSize=35.9 K 2024-11-23T16:16:22,892 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/A/f7c5657eeafc4f5197739834d768e9ba, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/A/88d5b8fb2b434b08a7306e7fb591eda5, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/A/46b30f27a7ed401da4d65c59ae93ef22] into tmpdir=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/.tmp, totalSize=91.4 K 2024-11-23T16:16:22,892 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36. 2024-11-23T16:16:22,892 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36. files: [hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/A/f7c5657eeafc4f5197739834d768e9ba, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/A/88d5b8fb2b434b08a7306e7fb591eda5, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/A/46b30f27a7ed401da4d65c59ae93ef22] 2024-11-23T16:16:22,893 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.Compactor(225): Compacting 79af371c8b3343d09fc762b6c3b3b392, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=140, earliestPutTs=1732378577642 2024-11-23T16:16:22,893 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.Compactor(225): Compacting f7c5657eeafc4f5197739834d768e9ba, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=140, earliestPutTs=1732378577642 2024-11-23T16:16:22,893 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.Compactor(225): Compacting dda8b67939ed4d189432c59113213813, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1732378578806 2024-11-23T16:16:22,893 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.Compactor(225): Compacting 88d5b8fb2b434b08a7306e7fb591eda5, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1732378578806 2024-11-23T16:16:22,893 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.Compactor(225): Compacting 6a7a82c2177441b3bd67e96df594a850, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=176, earliestPutTs=1732378580981 2024-11-23T16:16:22,893 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.Compactor(225): Compacting 46b30f27a7ed401da4d65c59ae93ef22, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=176, earliestPutTs=1732378580981 2024-11-23T16:16:22,906 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=1930901e64343eeb48a7930647197c36] 2024-11-23T16:16:22,907 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1930901e64343eeb48a7930647197c36#B#compaction#473 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-23T16:16:22,907 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/.tmp/B/8df8e5399fb443b88368bb162fa41a64 is 50, key is test_row_0/B:col10/1732378581618/Put/seqid=0 2024-11-23T16:16:22,921 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241123f1a8f84599cf4c4f90d8474493357243_1930901e64343eeb48a7930647197c36 store=[table=TestAcidGuarantees family=A region=1930901e64343eeb48a7930647197c36] 2024-11-23T16:16:22,923 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241123f1a8f84599cf4c4f90d8474493357243_1930901e64343eeb48a7930647197c36, store=[table=TestAcidGuarantees family=A region=1930901e64343eeb48a7930647197c36] 2024-11-23T16:16:22,923 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241123f1a8f84599cf4c4f90d8474493357243_1930901e64343eeb48a7930647197c36 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=1930901e64343eeb48a7930647197c36] 2024-11-23T16:16:22,951 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742387_1563 (size=12561) 2024-11-23T16:16:22,953 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742388_1564 (size=4469) 2024-11-23T16:16:22,955 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1930901e64343eeb48a7930647197c36#A#compaction#474 average throughput is 0.50 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T16:16:22,956 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/.tmp/A/1188308ed1a941c3b82febd0c0cf453b is 175, key is test_row_0/A:col10/1732378581618/Put/seqid=0 2024-11-23T16:16:22,963 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742389_1565 (size=31515) 2024-11-23T16:16:23,356 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/.tmp/B/8df8e5399fb443b88368bb162fa41a64 as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/B/8df8e5399fb443b88368bb162fa41a64 2024-11-23T16:16:23,360 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1930901e64343eeb48a7930647197c36/B of 1930901e64343eeb48a7930647197c36 into 8df8e5399fb443b88368bb162fa41a64(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T16:16:23,360 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1930901e64343eeb48a7930647197c36: 2024-11-23T16:16:23,360 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., storeName=1930901e64343eeb48a7930647197c36/B, priority=13, startTime=1732378582891; duration=0sec 2024-11-23T16:16:23,360 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T16:16:23,360 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1930901e64343eeb48a7930647197c36:B 2024-11-23T16:16:23,361 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T16:16:23,361 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T16:16:23,361 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HStore(1541): 1930901e64343eeb48a7930647197c36/C is initiating minor compaction (all files) 2024-11-23T16:16:23,361 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1930901e64343eeb48a7930647197c36/C in TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36. 2024-11-23T16:16:23,362 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/C/891b5a1cb3fb4bab82596b5bfe06ccfb, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/C/654750eb4f1b4b5494ab2e6869513f84, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/C/68d8914b3afd4f6fbbfbb65d84a1628f] into tmpdir=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/.tmp, totalSize=35.9 K 2024-11-23T16:16:23,362 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.Compactor(225): Compacting 891b5a1cb3fb4bab82596b5bfe06ccfb, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=140, earliestPutTs=1732378577642 2024-11-23T16:16:23,362 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.Compactor(225): Compacting 654750eb4f1b4b5494ab2e6869513f84, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1732378578806 2024-11-23T16:16:23,362 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.Compactor(225): Compacting 68d8914b3afd4f6fbbfbb65d84a1628f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=176, earliestPutTs=1732378580981 2024-11-23T16:16:23,367 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/.tmp/A/1188308ed1a941c3b82febd0c0cf453b as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/A/1188308ed1a941c3b82febd0c0cf453b 2024-11-23T16:16:23,369 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1930901e64343eeb48a7930647197c36#C#compaction#475 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T16:16:23,370 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/.tmp/C/7672978d77c44e29a7fd94b901538305 is 50, key is test_row_0/C:col10/1732378581618/Put/seqid=0 2024-11-23T16:16:23,372 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1930901e64343eeb48a7930647197c36/A of 1930901e64343eeb48a7930647197c36 into 1188308ed1a941c3b82febd0c0cf453b(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T16:16:23,372 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1930901e64343eeb48a7930647197c36: 2024-11-23T16:16:23,372 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., storeName=1930901e64343eeb48a7930647197c36/A, priority=13, startTime=1732378582891; duration=0sec 2024-11-23T16:16:23,372 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T16:16:23,372 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1930901e64343eeb48a7930647197c36:A 2024-11-23T16:16:23,387 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742390_1566 (size=12561) 2024-11-23T16:16:23,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-23T16:16:23,388 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-23T16:16:23,389 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45799 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-23T16:16:23,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45799 {}] procedure2.ProcedureExecutor(1139): Stored pid=148, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees 2024-11-23T16:16:23,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-23T16:16:23,390 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=148, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-23T16:16:23,393 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/.tmp/C/7672978d77c44e29a7fd94b901538305 as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/C/7672978d77c44e29a7fd94b901538305 2024-11-23T16:16:23,393 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=148, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-23T16:16:23,393 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=149, ppid=148, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-23T16:16:23,398 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1930901e64343eeb48a7930647197c36/C of 1930901e64343eeb48a7930647197c36 into 7672978d77c44e29a7fd94b901538305(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T16:16:23,398 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1930901e64343eeb48a7930647197c36: 2024-11-23T16:16:23,398 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., storeName=1930901e64343eeb48a7930647197c36/C, priority=13, startTime=1732378582891; duration=0sec 2024-11-23T16:16:23,399 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T16:16:23,399 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1930901e64343eeb48a7930647197c36:C 2024-11-23T16:16:23,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-23T16:16:23,545 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-23T16:16:23,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36. 2024-11-23T16:16:23,546 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2902): Flushing 1930901e64343eeb48a7930647197c36 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-23T16:16:23,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1930901e64343eeb48a7930647197c36, store=A 2024-11-23T16:16:23,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-23T16:16:23,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1930901e64343eeb48a7930647197c36, store=B 2024-11-23T16:16:23,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-23T16:16:23,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1930901e64343eeb48a7930647197c36, store=C 2024-11-23T16:16:23,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-23T16:16:23,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112361c766f75ca84e30b73aa163d0a544a7_1930901e64343eeb48a7930647197c36 is 50, key is test_row_0/A:col10/1732378581662/Put/seqid=0 2024-11-23T16:16:23,556 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742391_1567 (size=12304) 2024-11-23T16:16:23,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:23,560 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112361c766f75ca84e30b73aa163d0a544a7_1930901e64343eeb48a7930647197c36 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112361c766f75ca84e30b73aa163d0a544a7_1930901e64343eeb48a7930647197c36 2024-11-23T16:16:23,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/.tmp/A/295a9dd244a24539bc8c0785c393891d, store: [table=TestAcidGuarantees family=A region=1930901e64343eeb48a7930647197c36] 2024-11-23T16:16:23,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/.tmp/A/295a9dd244a24539bc8c0785c393891d is 175, key is test_row_0/A:col10/1732378581662/Put/seqid=0 2024-11-23T16:16:23,564 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742392_1568 (size=31105) 2024-11-23T16:16:23,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-23T16:16:23,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(8855): Flush requested on 1930901e64343eeb48a7930647197c36 2024-11-23T16:16:23,819 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36. as already flushing 2024-11-23T16:16:23,855 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:16:23,855 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:16:23,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53810 deadline: 1732378643850, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:16:23,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53830 deadline: 1732378643850, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:16:23,856 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:16:23,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53774 deadline: 1732378643851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:16:23,856 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:23,856 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:23,856 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:23,856 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:23,856 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:16:23,856 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:16:23,856 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:23,857 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:23,857 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:16:23,960 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:16:23,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53830 deadline: 1732378643957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:16:23,961 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:16:23,961 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:23,961 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:23,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53810 deadline: 1732378643958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:16:23,961 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:16:23,961 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:23,961 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:23,961 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:16:23,965 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=192, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/.tmp/A/295a9dd244a24539bc8c0785c393891d 2024-11-23T16:16:23,971 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:16:23,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53774 deadline: 1732378643968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:16:23,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/.tmp/B/fd0343b5c1f14747bc975805f9553146 is 50, key is test_row_0/B:col10/1732378581662/Put/seqid=0 2024-11-23T16:16:23,971 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:23,971 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:23,971 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:16:23,974 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742393_1569 (size=12151) 2024-11-23T16:16:24,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-23T16:16:24,170 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:16:24,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53830 deadline: 1732378644167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:16:24,171 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:16:24,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53810 deadline: 1732378644168, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:16:24,171 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:24,171 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:24,171 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:16:24,171 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:24,171 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:24,171 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:16:24,181 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:16:24,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53774 deadline: 1732378644178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:16:24,181 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:24,181 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:24,181 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:16:24,375 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=192 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/.tmp/B/fd0343b5c1f14747bc975805f9553146 2024-11-23T16:16:24,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/.tmp/C/2164692c3ca6456587a245be04e673fa is 50, key is test_row_0/C:col10/1732378581662/Put/seqid=0 2024-11-23T16:16:24,385 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742394_1570 (size=12151) 2024-11-23T16:16:24,480 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:16:24,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53830 deadline: 1732378644478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:16:24,481 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:24,481 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:16:24,481 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:24,481 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:16:24,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53810 deadline: 1732378644478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:16:24,481 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:24,481 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:24,481 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:16:24,489 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:16:24,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53774 deadline: 1732378644487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:16:24,489 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:24,490 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:24,490 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:16:24,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-23T16:16:24,785 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=192 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/.tmp/C/2164692c3ca6456587a245be04e673fa 2024-11-23T16:16:24,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/.tmp/A/295a9dd244a24539bc8c0785c393891d as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/A/295a9dd244a24539bc8c0785c393891d 2024-11-23T16:16:24,792 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/A/295a9dd244a24539bc8c0785c393891d, entries=150, sequenceid=192, filesize=30.4 K 2024-11-23T16:16:24,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/.tmp/B/fd0343b5c1f14747bc975805f9553146 as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/B/fd0343b5c1f14747bc975805f9553146 2024-11-23T16:16:24,796 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/B/fd0343b5c1f14747bc975805f9553146, entries=150, sequenceid=192, filesize=11.9 K 2024-11-23T16:16:24,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/.tmp/C/2164692c3ca6456587a245be04e673fa as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/C/2164692c3ca6456587a245be04e673fa 2024-11-23T16:16:24,799 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/C/2164692c3ca6456587a245be04e673fa, entries=150, sequenceid=192, filesize=11.9 K 2024-11-23T16:16:24,800 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 1930901e64343eeb48a7930647197c36 in 1254ms, sequenceid=192, compaction requested=false 2024-11-23T16:16:24,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2603): Flush status journal for 1930901e64343eeb48a7930647197c36: 2024-11-23T16:16:24,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36. 2024-11-23T16:16:24,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=149 2024-11-23T16:16:24,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45799 {}] master.HMaster(4169): Remote procedure done, pid=149 2024-11-23T16:16:24,803 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=149, resume processing ppid=148 2024-11-23T16:16:24,803 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=149, ppid=148, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4080 sec 2024-11-23T16:16:24,805 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=148, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees in 1.4140 sec 2024-11-23T16:16:24,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(8855): Flush requested on 1930901e64343eeb48a7930647197c36 2024-11-23T16:16:24,991 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1930901e64343eeb48a7930647197c36 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-23T16:16:24,991 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1930901e64343eeb48a7930647197c36, store=A 2024-11-23T16:16:24,991 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-23T16:16:24,991 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1930901e64343eeb48a7930647197c36, store=B 2024-11-23T16:16:24,991 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-23T16:16:24,991 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1930901e64343eeb48a7930647197c36, store=C 2024-11-23T16:16:24,991 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-23T16:16:24,998 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411232a6d625c2c82481da260d7a45c8fddbb_1930901e64343eeb48a7930647197c36 is 50, key is test_row_0/A:col10/1732378584989/Put/seqid=0 2024-11-23T16:16:25,001 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742395_1571 (size=14794) 2024-11-23T16:16:25,001 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:25,004 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411232a6d625c2c82481da260d7a45c8fddbb_1930901e64343eeb48a7930647197c36 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411232a6d625c2c82481da260d7a45c8fddbb_1930901e64343eeb48a7930647197c36 2024-11-23T16:16:25,005 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/.tmp/A/d8e6279db93b436182af0f4d8d6ac5e1, store: [table=TestAcidGuarantees family=A region=1930901e64343eeb48a7930647197c36] 2024-11-23T16:16:25,006 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/.tmp/A/d8e6279db93b436182af0f4d8d6ac5e1 is 175, key is test_row_0/A:col10/1732378584989/Put/seqid=0 2024-11-23T16:16:25,009 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742396_1572 (size=39749) 2024-11-23T16:16:25,012 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:16:25,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53830 deadline: 1732378645008, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:16:25,012 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:16:25,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53810 deadline: 1732378645009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:16:25,013 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:16:25,013 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:25,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53774 deadline: 1732378645010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:16:25,013 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:25,013 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:16:25,013 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:25,013 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:25,013 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:16:25,013 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:25,013 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:25,013 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:16:25,120 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:16:25,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53830 deadline: 1732378645118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:16:25,120 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:16:25,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53774 deadline: 1732378645118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:16:25,120 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:25,121 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:25,121 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:16:25,121 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:16:25,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53810 deadline: 1732378645118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:16:25,121 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:25,121 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:25,121 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:16:25,121 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:25,121 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:25,121 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:16:25,330 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:16:25,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53774 deadline: 1732378645328, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:16:25,331 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:16:25,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53830 deadline: 1732378645328, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:16:25,331 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:25,331 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:25,331 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:16:25,331 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:16:25,331 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:25,331 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:25,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53810 deadline: 1732378645328, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:16:25,331 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:16:25,331 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:25,331 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:25,331 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:16:25,410 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=212, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/.tmp/A/d8e6279db93b436182af0f4d8d6ac5e1 2024-11-23T16:16:25,416 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/.tmp/B/1ad4c0bb34fa4001ba0c2b8ad0ffc656 is 50, key is test_row_0/B:col10/1732378584989/Put/seqid=0 2024-11-23T16:16:25,419 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742397_1573 (size=12151) 2024-11-23T16:16:25,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-23T16:16:25,538 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-23T16:16:25,538 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45799 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-23T16:16:25,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45799 {}] procedure2.ProcedureExecutor(1139): Stored pid=150, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees 2024-11-23T16:16:25,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-23T16:16:25,540 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=150, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-23T16:16:25,541 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=150, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-23T16:16:25,541 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=151, ppid=150, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-23T16:16:25,639 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:16:25,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53774 deadline: 1732378645638, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:16:25,639 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:16:25,640 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:25,640 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:25,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53830 deadline: 1732378645638, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:16:25,640 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:16:25,640 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:16:25,640 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:25,640 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:25,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53810 deadline: 1732378645638, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:16:25,640 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:16:25,640 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:25,640 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:25,640 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:16:25,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-23T16:16:25,693 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-23T16:16:25,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36. 2024-11-23T16:16:25,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36. as already flushing 2024-11-23T16:16:25,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36. 2024-11-23T16:16:25,694 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 1930901e64343eeb48a7930647197c36, NAME => 'TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:16:25,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 1930901e64343eeb48a7930647197c36, NAME => 'TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:16:25,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45799 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1930901e64343eeb48a7930647197c36, NAME => 'TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1930901e64343eeb48a7930647197c36, NAME => 'TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:16:25,820 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=212 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/.tmp/B/1ad4c0bb34fa4001ba0c2b8ad0ffc656 2024-11-23T16:16:25,826 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/.tmp/C/d5f630abe14c4ed790930c2c46a9c90b is 50, key is test_row_0/C:col10/1732378584989/Put/seqid=0 2024-11-23T16:16:25,830 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742398_1574 (size=12151) 2024-11-23T16:16:25,847 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-23T16:16:25,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36. 2024-11-23T16:16:25,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36. as already flushing 2024-11-23T16:16:25,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36. 2024-11-23T16:16:25,847 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 1930901e64343eeb48a7930647197c36, NAME => 'TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:16:25,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 1930901e64343eeb48a7930647197c36, NAME => 'TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:16:25,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45799 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1930901e64343eeb48a7930647197c36, NAME => 'TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1930901e64343eeb48a7930647197c36, NAME => 'TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:16:25,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-23T16:16:26,000 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-23T16:16:26,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36. 2024-11-23T16:16:26,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36. as already flushing 2024-11-23T16:16:26,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36. 2024-11-23T16:16:26,000 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 1930901e64343eeb48a7930647197c36, NAME => 'TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:16:26,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 1930901e64343eeb48a7930647197c36, NAME => 'TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:16:26,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45799 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1930901e64343eeb48a7930647197c36, NAME => 'TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1930901e64343eeb48a7930647197c36, NAME => 'TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:16:26,152 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:16:26,152 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:16:26,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53774 deadline: 1732378646148, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:16:26,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53830 deadline: 1732378646148, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:16:26,152 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:16:26,152 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:26,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53810 deadline: 1732378646148, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:16:26,152 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:26,152 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:26,152 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:26,152 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:16:26,152 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:16:26,152 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:26,152 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:26,152 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:16:26,153 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-23T16:16:26,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36. 2024-11-23T16:16:26,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36. as already flushing 2024-11-23T16:16:26,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36. 2024-11-23T16:16:26,153 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 1930901e64343eeb48a7930647197c36, NAME => 'TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:16:26,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 1930901e64343eeb48a7930647197c36, NAME => 'TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:16:26,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45799 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1930901e64343eeb48a7930647197c36, NAME => 'TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1930901e64343eeb48a7930647197c36, NAME => 'TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:16:26,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-23T16:16:26,231 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=212 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/.tmp/C/d5f630abe14c4ed790930c2c46a9c90b 2024-11-23T16:16:26,234 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/.tmp/A/d8e6279db93b436182af0f4d8d6ac5e1 as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/A/d8e6279db93b436182af0f4d8d6ac5e1 2024-11-23T16:16:26,237 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/A/d8e6279db93b436182af0f4d8d6ac5e1, entries=200, sequenceid=212, filesize=38.8 K 2024-11-23T16:16:26,238 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/.tmp/B/1ad4c0bb34fa4001ba0c2b8ad0ffc656 as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/B/1ad4c0bb34fa4001ba0c2b8ad0ffc656 2024-11-23T16:16:26,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,241 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/B/1ad4c0bb34fa4001ba0c2b8ad0ffc656, entries=150, sequenceid=212, filesize=11.9 K 2024-11-23T16:16:26,241 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/.tmp/C/d5f630abe14c4ed790930c2c46a9c90b as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/C/d5f630abe14c4ed790930c2c46a9c90b 2024-11-23T16:16:26,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,245 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/C/d5f630abe14c4ed790930c2c46a9c90b, entries=150, sequenceid=212, filesize=11.9 K 2024-11-23T16:16:26,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,245 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 1930901e64343eeb48a7930647197c36 in 1255ms, sequenceid=212, compaction requested=true 2024-11-23T16:16:26,245 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1930901e64343eeb48a7930647197c36: 2024-11-23T16:16:26,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,246 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1930901e64343eeb48a7930647197c36:A, priority=-2147483648, current under compaction store size is 1 2024-11-23T16:16:26,246 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T16:16:26,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,246 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T16:16:26,246 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1930901e64343eeb48a7930647197c36:B, priority=-2147483648, current under compaction store size is 2 2024-11-23T16:16:26,246 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T16:16:26,246 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1930901e64343eeb48a7930647197c36:C, priority=-2147483648, current under compaction store size is 3 2024-11-23T16:16:26,246 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T16:16:26,246 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T16:16:26,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,247 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T16:16:26,247 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102369 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T16:16:26,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,247 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HStore(1541): 1930901e64343eeb48a7930647197c36/B is initiating minor compaction (all files) 2024-11-23T16:16:26,247 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HStore(1541): 1930901e64343eeb48a7930647197c36/A is initiating minor compaction (all files) 2024-11-23T16:16:26,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,247 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1930901e64343eeb48a7930647197c36/B in TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36. 2024-11-23T16:16:26,247 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1930901e64343eeb48a7930647197c36/A in TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36. 2024-11-23T16:16:26,247 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/B/8df8e5399fb443b88368bb162fa41a64, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/B/fd0343b5c1f14747bc975805f9553146, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/B/1ad4c0bb34fa4001ba0c2b8ad0ffc656] into tmpdir=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/.tmp, totalSize=36.0 K 2024-11-23T16:16:26,247 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/A/1188308ed1a941c3b82febd0c0cf453b, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/A/295a9dd244a24539bc8c0785c393891d, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/A/d8e6279db93b436182af0f4d8d6ac5e1] into tmpdir=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/.tmp, totalSize=100.0 K 2024-11-23T16:16:26,247 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36. 2024-11-23T16:16:26,247 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36. files: [hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/A/1188308ed1a941c3b82febd0c0cf453b, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/A/295a9dd244a24539bc8c0785c393891d, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/A/d8e6279db93b436182af0f4d8d6ac5e1] 2024-11-23T16:16:26,247 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.Compactor(225): Compacting 8df8e5399fb443b88368bb162fa41a64, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=176, earliestPutTs=1732378580981 2024-11-23T16:16:26,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,247 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1188308ed1a941c3b82febd0c0cf453b, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=176, earliestPutTs=1732378580981 2024-11-23T16:16:26,247 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.Compactor(225): Compacting fd0343b5c1f14747bc975805f9553146, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=192, earliestPutTs=1732378581656 2024-11-23T16:16:26,248 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.Compactor(225): Compacting 1ad4c0bb34fa4001ba0c2b8ad0ffc656, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1732378583836 2024-11-23T16:16:26,248 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.Compactor(225): Compacting 295a9dd244a24539bc8c0785c393891d, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=192, earliestPutTs=1732378581656 2024-11-23T16:16:26,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,248 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.Compactor(225): Compacting d8e6279db93b436182af0f4d8d6ac5e1, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1732378583836 2024-11-23T16:16:26,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,253 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=1930901e64343eeb48a7930647197c36] 2024-11-23T16:16:26,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,255 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1930901e64343eeb48a7930647197c36#B#compaction#483 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-23T16:16:26,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,256 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/.tmp/B/f0e40ac2d3564dbe8a3082e76342d115 is 50, key is test_row_0/B:col10/1732378584989/Put/seqid=0 2024-11-23T16:16:26,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,265 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241123a6fb63dcc020400aaba1ee86945a59c6_1930901e64343eeb48a7930647197c36 store=[table=TestAcidGuarantees family=A region=1930901e64343eeb48a7930647197c36] 2024-11-23T16:16:26,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,267 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241123a6fb63dcc020400aaba1ee86945a59c6_1930901e64343eeb48a7930647197c36, store=[table=TestAcidGuarantees family=A region=1930901e64343eeb48a7930647197c36] 2024-11-23T16:16:26,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,267 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241123a6fb63dcc020400aaba1ee86945a59c6_1930901e64343eeb48a7930647197c36 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=1930901e64343eeb48a7930647197c36] 2024-11-23T16:16:26,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,273 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742399_1575 (size=12663) 2024-11-23T16:16:26,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,278 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/.tmp/B/f0e40ac2d3564dbe8a3082e76342d115 as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/B/f0e40ac2d3564dbe8a3082e76342d115 2024-11-23T16:16:26,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,282 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1930901e64343eeb48a7930647197c36/B of 1930901e64343eeb48a7930647197c36 into f0e40ac2d3564dbe8a3082e76342d115(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T16:16:26,282 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1930901e64343eeb48a7930647197c36: 2024-11-23T16:16:26,282 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., storeName=1930901e64343eeb48a7930647197c36/B, priority=13, startTime=1732378586246; duration=0sec 2024-11-23T16:16:26,282 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T16:16:26,282 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1930901e64343eeb48a7930647197c36:B 2024-11-23T16:16:26,282 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T16:16:26,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,283 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T16:16:26,283 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HStore(1541): 1930901e64343eeb48a7930647197c36/C is initiating minor compaction (all files) 2024-11-23T16:16:26,283 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1930901e64343eeb48a7930647197c36/C in TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36. 2024-11-23T16:16:26,283 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/C/7672978d77c44e29a7fd94b901538305, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/C/2164692c3ca6456587a245be04e673fa, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/C/d5f630abe14c4ed790930c2c46a9c90b] into tmpdir=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/.tmp, totalSize=36.0 K 2024-11-23T16:16:26,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,284 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.Compactor(225): Compacting 7672978d77c44e29a7fd94b901538305, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=176, earliestPutTs=1732378580981 2024-11-23T16:16:26,284 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.Compactor(225): Compacting 2164692c3ca6456587a245be04e673fa, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=192, earliestPutTs=1732378581656 2024-11-23T16:16:26,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,284 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.Compactor(225): Compacting d5f630abe14c4ed790930c2c46a9c90b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1732378583836 2024-11-23T16:16:26,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,294 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742400_1576 (size=4469) 2024-11-23T16:16:26,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,301 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1930901e64343eeb48a7930647197c36#C#compaction#484 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-23T16:16:26,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,302 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/.tmp/C/3f0c45c085b24995861cacaedfaba9de is 50, key is test_row_0/C:col10/1732378584989/Put/seqid=0 2024-11-23T16:16:26,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,305 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742401_1577 (size=12663) 2024-11-23T16:16:26,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,305 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-23T16:16:26,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36. 2024-11-23T16:16:26,306 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2902): Flushing 1930901e64343eeb48a7930647197c36 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-23T16:16:26,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1930901e64343eeb48a7930647197c36, store=A 2024-11-23T16:16:26,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-23T16:16:26,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1930901e64343eeb48a7930647197c36, store=B 2024-11-23T16:16:26,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-23T16:16:26,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1930901e64343eeb48a7930647197c36, store=C 2024-11-23T16:16:26,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-23T16:16:26,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241123ec7eaebc3695467ca34d4f5a9de8945c_1930901e64343eeb48a7930647197c36 is 50, key is test_row_1/A:col10/1732378585003/Put/seqid=0 2024-11-23T16:16:26,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,322 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742402_1578 (size=9814) 2024-11-23T16:16:26,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,326 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241123ec7eaebc3695467ca34d4f5a9de8945c_1930901e64343eeb48a7930647197c36 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123ec7eaebc3695467ca34d4f5a9de8945c_1930901e64343eeb48a7930647197c36 2024-11-23T16:16:26,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/.tmp/A/6b31d0cb8b814bf7ace0849cafd2165e, store: [table=TestAcidGuarantees family=A region=1930901e64343eeb48a7930647197c36] 2024-11-23T16:16:26,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/.tmp/A/6b31d0cb8b814bf7ace0849cafd2165e is 175, key is test_row_1/A:col10/1732378585003/Put/seqid=0 2024-11-23T16:16:26,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,332 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742403_1579 (size=22461) 2024-11-23T16:16:26,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,333 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=225, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/.tmp/A/6b31d0cb8b814bf7ace0849cafd2165e 2024-11-23T16:16:26,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/.tmp/B/1cd046d016d54c2dafc646167a1bc893 is 50, key is test_row_1/B:col10/1732378585003/Put/seqid=0 2024-11-23T16:16:26,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,343 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742404_1580 (size=9757) 2024-11-23T16:16:26,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,344 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=225 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/.tmp/B/1cd046d016d54c2dafc646167a1bc893 2024-11-23T16:16:26,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/.tmp/C/64627dab0b494ba0be74bfaf9ce0a944 is 50, key is test_row_1/C:col10/1732378585003/Put/seqid=0 2024-11-23T16:16:26,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,362 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742405_1581 (size=9757) 2024-11-23T16:16:26,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-23T16:16:26,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,697 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1930901e64343eeb48a7930647197c36#A#compaction#482 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T16:16:26,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,698 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/.tmp/A/9a478905b9d04f659c11c62d3feee135 is 175, key is test_row_0/A:col10/1732378584989/Put/seqid=0 2024-11-23T16:16:26,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,703 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742406_1582 (size=31617) 2024-11-23T16:16:26,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,710 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/.tmp/C/3f0c45c085b24995861cacaedfaba9de as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/C/3f0c45c085b24995861cacaedfaba9de 2024-11-23T16:16:26,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,716 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1930901e64343eeb48a7930647197c36/C of 1930901e64343eeb48a7930647197c36 into 3f0c45c085b24995861cacaedfaba9de(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T16:16:26,717 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1930901e64343eeb48a7930647197c36: 2024-11-23T16:16:26,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,717 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., storeName=1930901e64343eeb48a7930647197c36/C, priority=13, startTime=1732378586246; duration=0sec 2024-11-23T16:16:26,717 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T16:16:26,717 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1930901e64343eeb48a7930647197c36:C 2024-11-23T16:16:26,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,763 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=225 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/.tmp/C/64627dab0b494ba0be74bfaf9ce0a944 2024-11-23T16:16:26,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/.tmp/A/6b31d0cb8b814bf7ace0849cafd2165e as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/A/6b31d0cb8b814bf7ace0849cafd2165e 2024-11-23T16:16:26,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,773 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/A/6b31d0cb8b814bf7ace0849cafd2165e, entries=100, sequenceid=225, filesize=21.9 K 2024-11-23T16:16:26,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/.tmp/B/1cd046d016d54c2dafc646167a1bc893 as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/B/1cd046d016d54c2dafc646167a1bc893 2024-11-23T16:16:26,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,778 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/B/1cd046d016d54c2dafc646167a1bc893, entries=100, sequenceid=225, filesize=9.5 K 2024-11-23T16:16:26,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/.tmp/C/64627dab0b494ba0be74bfaf9ce0a944 as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/C/64627dab0b494ba0be74bfaf9ce0a944 2024-11-23T16:16:26,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,782 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/C/64627dab0b494ba0be74bfaf9ce0a944, entries=100, sequenceid=225, filesize=9.5 K 2024-11-23T16:16:26,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,783 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=0 B/0 for 1930901e64343eeb48a7930647197c36 in 477ms, sequenceid=225, compaction requested=false 2024-11-23T16:16:26,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2603): Flush status journal for 1930901e64343eeb48a7930647197c36: 2024-11-23T16:16:26,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36. 2024-11-23T16:16:26,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=151 2024-11-23T16:16:26,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45799 {}] master.HMaster(4169): Remote procedure done, pid=151 2024-11-23T16:16:26,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,787 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=151, resume processing ppid=150 2024-11-23T16:16:26,787 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=151, ppid=150, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2440 sec 2024-11-23T16:16:26,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,789 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=150, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees in 1.2490 sec 2024-11-23T16:16:26,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:26,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,108 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/.tmp/A/9a478905b9d04f659c11c62d3feee135 as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/A/9a478905b9d04f659c11c62d3feee135 2024-11-23T16:16:27,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,114 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1930901e64343eeb48a7930647197c36/A of 1930901e64343eeb48a7930647197c36 into 9a478905b9d04f659c11c62d3feee135(size=30.9 K), total size for store is 52.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T16:16:27,114 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1930901e64343eeb48a7930647197c36: 2024-11-23T16:16:27,115 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., storeName=1930901e64343eeb48a7930647197c36/A, priority=13, startTime=1732378586246; duration=0sec 2024-11-23T16:16:27,115 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T16:16:27,115 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1930901e64343eeb48a7930647197c36:A 2024-11-23T16:16:27,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(8855): Flush requested on 1930901e64343eeb48a7930647197c36 2024-11-23T16:16:27,215 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1930901e64343eeb48a7930647197c36 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-23T16:16:27,215 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1930901e64343eeb48a7930647197c36, store=A 2024-11-23T16:16:27,215 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-23T16:16:27,215 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1930901e64343eeb48a7930647197c36, store=B 2024-11-23T16:16:27,215 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-23T16:16:27,215 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1930901e64343eeb48a7930647197c36, store=C 2024-11-23T16:16:27,215 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-23T16:16:27,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,223 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241123e6d7fa9be6a24ce185dde45fc0d96dba_1930901e64343eeb48a7930647197c36 is 50, key is test_row_0/A:col10/1732378587204/Put/seqid=0 2024-11-23T16:16:27,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,232 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742407_1583 (size=19774) 2024-11-23T16:16:27,236 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:27,240 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241123e6d7fa9be6a24ce185dde45fc0d96dba_1930901e64343eeb48a7930647197c36 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123e6d7fa9be6a24ce185dde45fc0d96dba_1930901e64343eeb48a7930647197c36 2024-11-23T16:16:27,240 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/.tmp/A/1ab0b906297a4b9a825c5dc4c1c239f7, store: [table=TestAcidGuarantees family=A region=1930901e64343eeb48a7930647197c36] 2024-11-23T16:16:27,241 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/.tmp/A/1ab0b906297a4b9a825c5dc4c1c239f7 is 175, key is test_row_0/A:col10/1732378587204/Put/seqid=0 2024-11-23T16:16:27,245 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742408_1584 (size=57033) 2024-11-23T16:16:27,287 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1a7c5470b472:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732378587233, max timestamp=1732378587283Num uniques 859; 2024-11-23T16:16:27,287 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1a7c5470b472:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732378587233, max timestamp=1732378587283Num uniques 859; 2024-11-23T16:16:27,287 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1a7c5470b472:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-23T16:16:27,287 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1a7c5470b472:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-23T16:16:27,287 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1a7c5470b472:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11854, totalHeapSize=27176, min timestamp=1732378587233, max timestamp=1732378587283Num uniques 859; 2024-11-23T16:16:27,287 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1a7c5470b472:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=11854, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-23T16:16:27,320 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:16:27,320 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:16:27,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53774 deadline: 1732378647313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:16:27,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53810 deadline: 1732378647313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:16:27,320 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:16:27,320 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:27,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53830 deadline: 1732378647314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:16:27,320 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:27,320 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:16:27,321 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:27,321 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:27,321 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:27,321 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:27,321 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:16:27,321 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:16:27,435 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:16:27,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53810 deadline: 1732378647433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:16:27,436 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:16:27,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53830 deadline: 1732378647433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:16:27,436 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:27,436 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:27,436 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:16:27,436 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:16:27,436 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:27,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53774 deadline: 1732378647433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:16:27,436 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:27,436 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:16:27,436 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:27,436 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:27,436 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:16:27,641 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:16:27,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53774 deadline: 1732378647638, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:16:27,641 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:27,641 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:27,641 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:16:27,646 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=237, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/.tmp/A/1ab0b906297a4b9a825c5dc4c1c239f7 2024-11-23T16:16:27,652 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:16:27,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53810 deadline: 1732378647648, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:16:27,652 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:16:27,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53830 deadline: 1732378647648, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:16:27,652 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:27,652 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:27,652 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:16:27,653 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/.tmp/B/76f499efa7464821a3fa94a888af24cd is 50, key is test_row_0/B:col10/1732378587204/Put/seqid=0 2024-11-23T16:16:27,653 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:27,653 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:27,653 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:16:27,656 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742409_1585 (size=12151) 2024-11-23T16:16:27,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-23T16:16:27,698 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-23T16:16:27,698 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45799 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-23T16:16:27,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45799 {}] procedure2.ProcedureExecutor(1139): Stored pid=152, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees 2024-11-23T16:16:27,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-23T16:16:27,700 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=152, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-23T16:16:27,701 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=152, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-23T16:16:27,701 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=153, ppid=152, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-23T16:16:27,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-23T16:16:27,853 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-23T16:16:27,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36. 2024-11-23T16:16:27,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36. as already flushing 2024-11-23T16:16:27,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36. 2024-11-23T16:16:27,853 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 1930901e64343eeb48a7930647197c36, NAME => 'TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:16:27,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 1930901e64343eeb48a7930647197c36, NAME => 'TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:16:27,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45799 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1930901e64343eeb48a7930647197c36, NAME => 'TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1930901e64343eeb48a7930647197c36, NAME => 'TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:16:27,950 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:16:27,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53774 deadline: 1732378647947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:16:27,950 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:27,951 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:27,951 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:16:27,961 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:16:27,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53810 deadline: 1732378647958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:16:27,961 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:16:27,961 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:27,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53830 deadline: 1732378647958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:16:27,961 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:27,961 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:16:27,961 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:27,961 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:27,961 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:16:28,006 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-23T16:16:28,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36. 2024-11-23T16:16:28,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36. as already flushing 2024-11-23T16:16:28,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36. 2024-11-23T16:16:28,006 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 1930901e64343eeb48a7930647197c36, NAME => 'TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:16:28,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 1930901e64343eeb48a7930647197c36, NAME => 'TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:16:28,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45799 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1930901e64343eeb48a7930647197c36, NAME => 'TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1930901e64343eeb48a7930647197c36, NAME => 'TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:16:28,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-23T16:16:28,056 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=237 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/.tmp/B/76f499efa7464821a3fa94a888af24cd 2024-11-23T16:16:28,063 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/.tmp/C/9fe451c28dbe4cba892282eefb819879 is 50, key is test_row_0/C:col10/1732378587204/Put/seqid=0 2024-11-23T16:16:28,066 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742410_1586 (size=12151) 2024-11-23T16:16:28,159 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-23T16:16:28,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36. 2024-11-23T16:16:28,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36. as already flushing 2024-11-23T16:16:28,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36. 2024-11-23T16:16:28,159 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 1930901e64343eeb48a7930647197c36, NAME => 'TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:16:28,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 1930901e64343eeb48a7930647197c36, NAME => 'TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:16:28,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45799 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1930901e64343eeb48a7930647197c36, NAME => 'TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1930901e64343eeb48a7930647197c36, NAME => 'TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:16:28,312 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-23T16:16:28,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36. 2024-11-23T16:16:28,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36. as already flushing 2024-11-23T16:16:28,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36. 2024-11-23T16:16:28,312 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 1930901e64343eeb48a7930647197c36, NAME => 'TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:16:28,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 1930901e64343eeb48a7930647197c36, NAME => 'TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:16:28,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45799 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1930901e64343eeb48a7930647197c36, NAME => 'TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1930901e64343eeb48a7930647197c36, NAME => 'TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:16:28,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-23T16:16:28,462 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:16:28,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53774 deadline: 1732378648458, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:16:28,463 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:28,463 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:28,463 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:16:28,465 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-23T16:16:28,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36. 2024-11-23T16:16:28,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36. as already flushing 2024-11-23T16:16:28,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36. 2024-11-23T16:16:28,465 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 1930901e64343eeb48a7930647197c36, NAME => 'TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:16:28,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 1930901e64343eeb48a7930647197c36, NAME => 'TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:16:28,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45799 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1930901e64343eeb48a7930647197c36, NAME => 'TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1930901e64343eeb48a7930647197c36, NAME => 'TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T16:16:28,467 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=237 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/.tmp/C/9fe451c28dbe4cba892282eefb819879 2024-11-23T16:16:28,471 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/.tmp/A/1ab0b906297a4b9a825c5dc4c1c239f7 as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/A/1ab0b906297a4b9a825c5dc4c1c239f7 2024-11-23T16:16:28,472 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:16:28,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53810 deadline: 1732378648468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:16:28,473 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:28,473 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:28,473 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:16:28,473 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T16:16:28,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44715 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53830 deadline: 1732378648468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 2024-11-23T16:16:28,473 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 , the old value is region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:28,473 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T16:16:28,473 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., hostname=1a7c5470b472,44715,1732378420375, seqNum=5 because the exception is null or not the one we care about 2024-11-23T16:16:28,474 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/A/1ab0b906297a4b9a825c5dc4c1c239f7, entries=300, sequenceid=237, filesize=55.7 K 2024-11-23T16:16:28,475 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/.tmp/B/76f499efa7464821a3fa94a888af24cd as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/B/76f499efa7464821a3fa94a888af24cd 2024-11-23T16:16:28,478 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/B/76f499efa7464821a3fa94a888af24cd, entries=150, sequenceid=237, filesize=11.9 K 2024-11-23T16:16:28,478 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/.tmp/C/9fe451c28dbe4cba892282eefb819879 as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/C/9fe451c28dbe4cba892282eefb819879 2024-11-23T16:16:28,482 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/C/9fe451c28dbe4cba892282eefb819879, entries=150, sequenceid=237, filesize=11.9 K 2024-11-23T16:16:28,482 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 1930901e64343eeb48a7930647197c36 in 1267ms, sequenceid=237, compaction requested=true 2024-11-23T16:16:28,482 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1930901e64343eeb48a7930647197c36: 2024-11-23T16:16:28,482 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1930901e64343eeb48a7930647197c36:A, priority=-2147483648, current under compaction store size is 1 2024-11-23T16:16:28,482 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T16:16:28,482 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1930901e64343eeb48a7930647197c36:B, priority=-2147483648, current under compaction store size is 2 2024-11-23T16:16:28,482 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T16:16:28,482 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T16:16:28,483 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1930901e64343eeb48a7930647197c36:C, priority=-2147483648, current under compaction store size is 3 2024-11-23T16:16:28,483 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-23T16:16:28,483 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T16:16:28,483 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 111111 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T16:16:28,483 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34571 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T16:16:28,483 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HStore(1541): 1930901e64343eeb48a7930647197c36/B is initiating minor compaction (all files) 2024-11-23T16:16:28,483 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HStore(1541): 1930901e64343eeb48a7930647197c36/A is initiating minor compaction (all files) 2024-11-23T16:16:28,484 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1930901e64343eeb48a7930647197c36/B in TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36. 2024-11-23T16:16:28,484 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1930901e64343eeb48a7930647197c36/A in TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36. 2024-11-23T16:16:28,484 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/A/9a478905b9d04f659c11c62d3feee135, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/A/6b31d0cb8b814bf7ace0849cafd2165e, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/A/1ab0b906297a4b9a825c5dc4c1c239f7] into tmpdir=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/.tmp, totalSize=108.5 K 2024-11-23T16:16:28,484 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/B/f0e40ac2d3564dbe8a3082e76342d115, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/B/1cd046d016d54c2dafc646167a1bc893, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/B/76f499efa7464821a3fa94a888af24cd] into tmpdir=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/.tmp, totalSize=33.8 K 2024-11-23T16:16:28,484 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36. 2024-11-23T16:16:28,484 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36. files: [hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/A/9a478905b9d04f659c11c62d3feee135, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/A/6b31d0cb8b814bf7ace0849cafd2165e, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/A/1ab0b906297a4b9a825c5dc4c1c239f7] 2024-11-23T16:16:28,484 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.Compactor(225): Compacting f0e40ac2d3564dbe8a3082e76342d115, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1732378583836 2024-11-23T16:16:28,484 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9a478905b9d04f659c11c62d3feee135, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1732378583836 2024-11-23T16:16:28,484 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.Compactor(225): Compacting 1cd046d016d54c2dafc646167a1bc893, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=225, earliestPutTs=1732378585003 2024-11-23T16:16:28,484 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6b31d0cb8b814bf7ace0849cafd2165e, keycount=100, bloomtype=ROW, size=21.9 K, encoding=NONE, compression=NONE, seqNum=225, earliestPutTs=1732378585003 2024-11-23T16:16:28,485 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.Compactor(225): Compacting 76f499efa7464821a3fa94a888af24cd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=237, earliestPutTs=1732378587204 2024-11-23T16:16:28,485 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1ab0b906297a4b9a825c5dc4c1c239f7, keycount=300, bloomtype=ROW, size=55.7 K, encoding=NONE, compression=NONE, seqNum=237, earliestPutTs=1732378587186 2024-11-23T16:16:28,490 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1930901e64343eeb48a7930647197c36#B#compaction#491 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T16:16:28,491 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/.tmp/B/6ed9e8bd0e3f4d63a13da6131f60c1fb is 50, key is test_row_0/B:col10/1732378587204/Put/seqid=0 2024-11-23T16:16:28,491 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=1930901e64343eeb48a7930647197c36] 2024-11-23T16:16:28,493 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241123c19cedba49e841caa46ce092459455fa_1930901e64343eeb48a7930647197c36 store=[table=TestAcidGuarantees family=A region=1930901e64343eeb48a7930647197c36] 2024-11-23T16:16:28,495 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241123c19cedba49e841caa46ce092459455fa_1930901e64343eeb48a7930647197c36, store=[table=TestAcidGuarantees family=A region=1930901e64343eeb48a7930647197c36] 2024-11-23T16:16:28,495 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241123c19cedba49e841caa46ce092459455fa_1930901e64343eeb48a7930647197c36 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=1930901e64343eeb48a7930647197c36] 2024-11-23T16:16:28,496 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742411_1587 (size=12765) 2024-11-23T16:16:28,498 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742412_1588 (size=4469) 2024-11-23T16:16:28,618 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-23T16:16:28,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36. 2024-11-23T16:16:28,618 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2902): Flushing 1930901e64343eeb48a7930647197c36 3/3 column families, dataSize=134.18 KB heapSize=375.61 KB 2024-11-23T16:16:28,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1930901e64343eeb48a7930647197c36, store=A 2024-11-23T16:16:28,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-23T16:16:28,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1930901e64343eeb48a7930647197c36, store=B 2024-11-23T16:16:28,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-23T16:16:28,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1930901e64343eeb48a7930647197c36, store=C 2024-11-23T16:16:28,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-23T16:16:28,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411234827fee1747f446cb0c20bac3a531005_1930901e64343eeb48a7930647197c36 is 50, key is test_row_0/A:col10/1732378587312/Put/seqid=0 2024-11-23T16:16:28,629 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742413_1589 (size=12454) 2024-11-23T16:16:28,753 INFO [Thread-2313 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2313. 2024-11-23T16:16:28,753 DEBUG [Thread-2313 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-23T16:16:28,753 DEBUG [Thread-2313 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T16:16:28,753 DEBUG [Thread-2313 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T16:16:28,753 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-23T16:16:28,755 INFO [Thread-2317 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2317. 2024-11-23T16:16:28,755 INFO [Thread-2319 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2319. 2024-11-23T16:16:28,755 DEBUG [Thread-2317 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-23T16:16:28,755 DEBUG [Thread-2317 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T16:16:28,755 DEBUG [Thread-2319 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-23T16:16:28,755 DEBUG [Thread-2319 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T16:16:28,755 DEBUG [Thread-2317 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T16:16:28,755 DEBUG [Thread-2319 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T16:16:28,755 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-23T16:16:28,755 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-23T16:16:28,757 INFO [Thread-2315 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2315. 2024-11-23T16:16:28,757 DEBUG [Thread-2315 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-23T16:16:28,757 DEBUG [Thread-2315 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T16:16:28,757 DEBUG [Thread-2315 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T16:16:28,757 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-23T16:16:28,758 INFO [Thread-2321 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2321. 2024-11-23T16:16:28,758 DEBUG [Thread-2321 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-23T16:16:28,758 DEBUG [Thread-2321 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T16:16:28,758 DEBUG [Thread-2321 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T16:16:28,758 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-23T16:16:28,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-23T16:16:28,899 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1930901e64343eeb48a7930647197c36#A#compaction#492 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T16:16:28,899 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/.tmp/A/6c0b5249fc794e639df6f9257b284b33 is 175, key is test_row_0/A:col10/1732378587204/Put/seqid=0 2024-11-23T16:16:28,900 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/.tmp/B/6ed9e8bd0e3f4d63a13da6131f60c1fb as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/B/6ed9e8bd0e3f4d63a13da6131f60c1fb 2024-11-23T16:16:28,902 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742414_1590 (size=31719) 2024-11-23T16:16:28,903 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1930901e64343eeb48a7930647197c36/B of 1930901e64343eeb48a7930647197c36 into 6ed9e8bd0e3f4d63a13da6131f60c1fb(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T16:16:28,903 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1930901e64343eeb48a7930647197c36: 2024-11-23T16:16:28,904 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., storeName=1930901e64343eeb48a7930647197c36/B, priority=13, startTime=1732378588482; duration=0sec 2024-11-23T16:16:28,904 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T16:16:28,904 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1930901e64343eeb48a7930647197c36:B 2024-11-23T16:16:28,904 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T16:16:28,904 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34571 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T16:16:28,904 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HStore(1541): 1930901e64343eeb48a7930647197c36/C is initiating minor compaction (all files) 2024-11-23T16:16:28,904 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1930901e64343eeb48a7930647197c36/C in TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36. 2024-11-23T16:16:28,904 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/C/3f0c45c085b24995861cacaedfaba9de, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/C/64627dab0b494ba0be74bfaf9ce0a944, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/C/9fe451c28dbe4cba892282eefb819879] into tmpdir=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/.tmp, totalSize=33.8 K 2024-11-23T16:16:28,905 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.Compactor(225): Compacting 3f0c45c085b24995861cacaedfaba9de, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1732378583836 2024-11-23T16:16:28,905 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.Compactor(225): Compacting 64627dab0b494ba0be74bfaf9ce0a944, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=225, earliestPutTs=1732378585003 2024-11-23T16:16:28,905 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] compactions.Compactor(225): Compacting 9fe451c28dbe4cba892282eefb819879, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=237, earliestPutTs=1732378587204 2024-11-23T16:16:28,909 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1930901e64343eeb48a7930647197c36#C#compaction#494 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T16:16:28,910 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/.tmp/C/457eb4fcb859406d89c66337ce2fb7d6 is 50, key is test_row_0/C:col10/1732378587204/Put/seqid=0 2024-11-23T16:16:28,912 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742415_1591 (size=12765) 2024-11-23T16:16:29,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:29,033 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411234827fee1747f446cb0c20bac3a531005_1930901e64343eeb48a7930647197c36 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411234827fee1747f446cb0c20bac3a531005_1930901e64343eeb48a7930647197c36 2024-11-23T16:16:29,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/.tmp/A/81fcc72f78b9437c8b1f0740beb6c2f0, store: [table=TestAcidGuarantees family=A region=1930901e64343eeb48a7930647197c36] 2024-11-23T16:16:29,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/.tmp/A/81fcc72f78b9437c8b1f0740beb6c2f0 is 175, key is test_row_0/A:col10/1732378587312/Put/seqid=0 2024-11-23T16:16:29,037 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742416_1592 (size=31255) 2024-11-23T16:16:29,307 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/.tmp/A/6c0b5249fc794e639df6f9257b284b33 as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/A/6c0b5249fc794e639df6f9257b284b33 2024-11-23T16:16:29,310 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1930901e64343eeb48a7930647197c36/A of 1930901e64343eeb48a7930647197c36 into 6c0b5249fc794e639df6f9257b284b33(size=31.0 K), total size for store is 31.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T16:16:29,310 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1930901e64343eeb48a7930647197c36: 2024-11-23T16:16:29,310 INFO [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., storeName=1930901e64343eeb48a7930647197c36/A, priority=13, startTime=1732378588482; duration=0sec 2024-11-23T16:16:29,310 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T16:16:29,310 DEBUG [RS:0;1a7c5470b472:44715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1930901e64343eeb48a7930647197c36:A 2024-11-23T16:16:29,315 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/.tmp/C/457eb4fcb859406d89c66337ce2fb7d6 as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/C/457eb4fcb859406d89c66337ce2fb7d6 2024-11-23T16:16:29,318 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1930901e64343eeb48a7930647197c36/C of 1930901e64343eeb48a7930647197c36 into 457eb4fcb859406d89c66337ce2fb7d6(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T16:16:29,318 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1930901e64343eeb48a7930647197c36: 2024-11-23T16:16:29,318 INFO [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., storeName=1930901e64343eeb48a7930647197c36/C, priority=13, startTime=1732378588482; duration=0sec 2024-11-23T16:16:29,318 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T16:16:29,318 DEBUG [RS:0;1a7c5470b472:44715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1930901e64343eeb48a7930647197c36:C 2024-11-23T16:16:29,438 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=272, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/.tmp/A/81fcc72f78b9437c8b1f0740beb6c2f0 2024-11-23T16:16:29,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/.tmp/B/01de1b3aba2b4eac875844ab5f24c5c4 is 50, key is test_row_0/B:col10/1732378587312/Put/seqid=0 2024-11-23T16:16:29,446 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742417_1593 (size=12301) 2024-11-23T16:16:29,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44715 {}] regionserver.HRegion(8855): Flush requested on 1930901e64343eeb48a7930647197c36 2024-11-23T16:16:29,469 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36. as already flushing 2024-11-23T16:16:29,469 INFO [Thread-2306 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2306. 2024-11-23T16:16:29,469 DEBUG [Thread-2306 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-23T16:16:29,469 DEBUG [Thread-2306 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T16:16:29,469 DEBUG [Thread-2306 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T16:16:29,469 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-23T16:16:29,479 INFO [Thread-2304 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2304. 2024-11-23T16:16:29,479 DEBUG [Thread-2304 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-23T16:16:29,479 DEBUG [Thread-2304 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T16:16:29,479 DEBUG [Thread-2304 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T16:16:29,479 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-23T16:16:29,489 INFO [Thread-2308 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2308. 2024-11-23T16:16:29,489 DEBUG [Thread-2308 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-23T16:16:29,489 DEBUG [Thread-2308 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T16:16:29,489 DEBUG [Thread-2308 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T16:16:29,489 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-23T16:16:29,847 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=272 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/.tmp/B/01de1b3aba2b4eac875844ab5f24c5c4 2024-11-23T16:16:29,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-23T16:16:29,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/.tmp/C/4d49dafa561d447d8398fab71bfee5a6 is 50, key is test_row_0/C:col10/1732378587312/Put/seqid=0 2024-11-23T16:16:29,855 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742418_1594 (size=12301) 2024-11-23T16:16:30,256 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=272 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/.tmp/C/4d49dafa561d447d8398fab71bfee5a6 2024-11-23T16:16:30,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/.tmp/A/81fcc72f78b9437c8b1f0740beb6c2f0 as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/A/81fcc72f78b9437c8b1f0740beb6c2f0 2024-11-23T16:16:30,262 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/A/81fcc72f78b9437c8b1f0740beb6c2f0, entries=150, sequenceid=272, filesize=30.5 K 2024-11-23T16:16:30,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/.tmp/B/01de1b3aba2b4eac875844ab5f24c5c4 as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/B/01de1b3aba2b4eac875844ab5f24c5c4 2024-11-23T16:16:30,265 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/B/01de1b3aba2b4eac875844ab5f24c5c4, entries=150, sequenceid=272, filesize=12.0 K 2024-11-23T16:16:30,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/.tmp/C/4d49dafa561d447d8398fab71bfee5a6 as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/C/4d49dafa561d447d8398fab71bfee5a6 2024-11-23T16:16:30,267 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/C/4d49dafa561d447d8398fab71bfee5a6, entries=150, sequenceid=272, filesize=12.0 K 2024-11-23T16:16:30,268 INFO [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~375.56 KB/384576, currentSize=20.13 KB/20610 for 1930901e64343eeb48a7930647197c36 in 1650ms, sequenceid=272, compaction requested=false 2024-11-23T16:16:30,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2603): Flush status journal for 1930901e64343eeb48a7930647197c36: 2024-11-23T16:16:30,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36. 2024-11-23T16:16:30,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7c5470b472:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=153 2024-11-23T16:16:30,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45799 {}] master.HMaster(4169): Remote procedure done, pid=153 2024-11-23T16:16:30,271 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=153, resume processing ppid=152 2024-11-23T16:16:30,271 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=153, ppid=152, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.5690 sec 2024-11-23T16:16:30,272 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=152, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees in 2.5730 sec 2024-11-23T16:16:30,370 INFO [Thread-2302 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2302. 2024-11-23T16:16:30,370 DEBUG [Thread-2302 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-23T16:16:30,370 DEBUG [Thread-2302 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T16:16:30,370 DEBUG [Thread-2302 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T16:16:30,370 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-23T16:16:30,389 INFO [Thread-2310 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2310. 2024-11-23T16:16:30,389 DEBUG [Thread-2310 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-23T16:16:30,389 DEBUG [Thread-2310 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T16:16:30,389 DEBUG [Thread-2310 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T16:16:30,389 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-23T16:16:31,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-23T16:16:31,858 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-23T16:16:31,858 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-23T16:16:31,858 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 16 2024-11-23T16:16:31,858 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 57 2024-11-23T16:16:31,858 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 62 2024-11-23T16:16:31,858 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 64 2024-11-23T16:16:31,858 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 17 2024-11-23T16:16:31,858 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-23T16:16:31,858 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3352 2024-11-23T16:16:31,858 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3084 2024-11-23T16:16:31,858 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3336 2024-11-23T16:16:31,858 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3365 2024-11-23T16:16:31,858 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3084 2024-11-23T16:16:31,858 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-23T16:16:31,858 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-23T16:16:31,858 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobGetAtomicity(AcidGuaranteesTestBase.java:122) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-23T16:16:31,858 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T16:16:31,859 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T16:16:31,859 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-23T16:16:31,859 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45799 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-23T16:16:31,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45799 {}] procedure2.ProcedureExecutor(1139): Stored pid=154, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-23T16:16:31,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-23T16:16:31,862 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732378591862"}]},"ts":"1732378591862"} 2024-11-23T16:16:31,863 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-23T16:16:31,863 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-23T16:16:31,864 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=155, ppid=154, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-23T16:16:31,865 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=156, ppid=155, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1930901e64343eeb48a7930647197c36, UNASSIGN}] 2024-11-23T16:16:31,866 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=156, ppid=155, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1930901e64343eeb48a7930647197c36, UNASSIGN 2024-11-23T16:16:31,866 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=156 updating hbase:meta row=1930901e64343eeb48a7930647197c36, regionState=CLOSING, regionLocation=1a7c5470b472,44715,1732378420375 2024-11-23T16:16:31,867 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=156, ppid=155, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=1930901e64343eeb48a7930647197c36, UNASSIGN because future has completed 2024-11-23T16:16:31,868 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-23T16:16:31,868 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=157, ppid=156, state=RUNNABLE, hasLock=false; CloseRegionProcedure 1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375}] 2024-11-23T16:16:31,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-23T16:16:32,020 INFO [RS_CLOSE_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_CLOSE_REGION, pid=157}] handler.UnassignRegionHandler(122): Close 1930901e64343eeb48a7930647197c36 2024-11-23T16:16:32,020 DEBUG [RS_CLOSE_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_CLOSE_REGION, pid=157}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-23T16:16:32,020 DEBUG [RS_CLOSE_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_CLOSE_REGION, pid=157}] regionserver.HRegion(1722): Closing 1930901e64343eeb48a7930647197c36, disabling compactions & flushes 2024-11-23T16:16:32,020 INFO [RS_CLOSE_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_CLOSE_REGION, pid=157}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36. 2024-11-23T16:16:32,020 DEBUG [RS_CLOSE_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_CLOSE_REGION, pid=157}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36. 2024-11-23T16:16:32,020 DEBUG [RS_CLOSE_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_CLOSE_REGION, pid=157}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36. after waiting 0 ms 2024-11-23T16:16:32,020 DEBUG [RS_CLOSE_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_CLOSE_REGION, pid=157}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36. 2024-11-23T16:16:32,020 INFO [RS_CLOSE_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_CLOSE_REGION, pid=157}] regionserver.HRegion(2902): Flushing 1930901e64343eeb48a7930647197c36 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-23T16:16:32,021 DEBUG [RS_CLOSE_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_CLOSE_REGION, pid=157}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1930901e64343eeb48a7930647197c36, store=A 2024-11-23T16:16:32,021 DEBUG [RS_CLOSE_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_CLOSE_REGION, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-23T16:16:32,021 DEBUG [RS_CLOSE_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_CLOSE_REGION, pid=157}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1930901e64343eeb48a7930647197c36, store=B 2024-11-23T16:16:32,021 DEBUG [RS_CLOSE_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_CLOSE_REGION, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-23T16:16:32,021 DEBUG [RS_CLOSE_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_CLOSE_REGION, pid=157}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1930901e64343eeb48a7930647197c36, store=C 2024-11-23T16:16:32,021 DEBUG [RS_CLOSE_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_CLOSE_REGION, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-23T16:16:32,027 DEBUG [RS_CLOSE_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_CLOSE_REGION, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112308bab72003394a44b9e7bcbb682d6a07_1930901e64343eeb48a7930647197c36 is 50, key is test_row_1/A:col10/1732378589488/Put/seqid=0 2024-11-23T16:16:32,037 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742419_1595 (size=9914) 2024-11-23T16:16:32,040 DEBUG [RS_CLOSE_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_CLOSE_REGION, pid=157}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T16:16:32,043 INFO [RS_CLOSE_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_CLOSE_REGION, pid=157}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112308bab72003394a44b9e7bcbb682d6a07_1930901e64343eeb48a7930647197c36 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112308bab72003394a44b9e7bcbb682d6a07_1930901e64343eeb48a7930647197c36 2024-11-23T16:16:32,043 DEBUG [RS_CLOSE_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_CLOSE_REGION, pid=157}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/.tmp/A/9ee67db43b4a48c4ac0843613e404127, store: [table=TestAcidGuarantees family=A region=1930901e64343eeb48a7930647197c36] 2024-11-23T16:16:32,044 DEBUG [RS_CLOSE_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_CLOSE_REGION, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/.tmp/A/9ee67db43b4a48c4ac0843613e404127 is 175, key is test_row_1/A:col10/1732378589488/Put/seqid=0 2024-11-23T16:16:32,051 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742420_1596 (size=22561) 2024-11-23T16:16:32,052 INFO [RS_CLOSE_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_CLOSE_REGION, pid=157}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=283, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/.tmp/A/9ee67db43b4a48c4ac0843613e404127 2024-11-23T16:16:32,058 DEBUG [RS_CLOSE_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_CLOSE_REGION, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/.tmp/B/fab1902dcf84438c9a0cda1efb79744e is 50, key is test_row_1/B:col10/1732378589488/Put/seqid=0 2024-11-23T16:16:32,061 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742421_1597 (size=9857) 2024-11-23T16:16:32,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-23T16:16:32,462 INFO [RS_CLOSE_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_CLOSE_REGION, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=283 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/.tmp/B/fab1902dcf84438c9a0cda1efb79744e 2024-11-23T16:16:32,467 DEBUG [RS_CLOSE_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_CLOSE_REGION, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/.tmp/C/5cb134a3dbd049c79551f9a19b0489db is 50, key is test_row_1/C:col10/1732378589488/Put/seqid=0 2024-11-23T16:16:32,470 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742422_1598 (size=9857) 2024-11-23T16:16:32,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-23T16:16:32,871 INFO [RS_CLOSE_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_CLOSE_REGION, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=283 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/.tmp/C/5cb134a3dbd049c79551f9a19b0489db 2024-11-23T16:16:32,874 DEBUG [RS_CLOSE_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_CLOSE_REGION, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/.tmp/A/9ee67db43b4a48c4ac0843613e404127 as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/A/9ee67db43b4a48c4ac0843613e404127 2024-11-23T16:16:32,881 INFO [RS_CLOSE_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_CLOSE_REGION, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/A/9ee67db43b4a48c4ac0843613e404127, entries=100, sequenceid=283, filesize=22.0 K 2024-11-23T16:16:32,882 DEBUG [RS_CLOSE_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_CLOSE_REGION, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/.tmp/B/fab1902dcf84438c9a0cda1efb79744e as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/B/fab1902dcf84438c9a0cda1efb79744e 2024-11-23T16:16:32,885 INFO [RS_CLOSE_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_CLOSE_REGION, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/B/fab1902dcf84438c9a0cda1efb79744e, entries=100, sequenceid=283, filesize=9.6 K 2024-11-23T16:16:32,885 DEBUG [RS_CLOSE_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_CLOSE_REGION, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/.tmp/C/5cb134a3dbd049c79551f9a19b0489db as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/C/5cb134a3dbd049c79551f9a19b0489db 2024-11-23T16:16:32,889 INFO [RS_CLOSE_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_CLOSE_REGION, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/C/5cb134a3dbd049c79551f9a19b0489db, entries=100, sequenceid=283, filesize=9.6 K 2024-11-23T16:16:32,890 INFO [RS_CLOSE_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_CLOSE_REGION, pid=157}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 1930901e64343eeb48a7930647197c36 in 870ms, sequenceid=283, compaction requested=true 2024-11-23T16:16:32,890 DEBUG [StoreCloser-TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/A/b1c124a4b0a34192aaaace777e6da0f1, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/A/19b2fe1b7c5a42cf9a642083b4edf826, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/A/9032ef42731641baba6d8904f48110c4, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/A/d1ab2ef581cc419bb126c47d56b4e937, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/A/7a7b39e3beb34eb08f2acffbbfa21342, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/A/6d967083128640b8a90cbd86180672b3, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/A/58f564b66ed54635a5e31adb9aeba85f, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/A/3f5f1c6d7b4e43e28da90dcb2b10f617, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/A/f7c5657eeafc4f5197739834d768e9ba, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/A/fcea5c7418154dffac3ea561659c88e7, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/A/88d5b8fb2b434b08a7306e7fb591eda5, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/A/1188308ed1a941c3b82febd0c0cf453b, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/A/46b30f27a7ed401da4d65c59ae93ef22, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/A/295a9dd244a24539bc8c0785c393891d, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/A/d8e6279db93b436182af0f4d8d6ac5e1, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/A/9a478905b9d04f659c11c62d3feee135, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/A/6b31d0cb8b814bf7ace0849cafd2165e, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/A/1ab0b906297a4b9a825c5dc4c1c239f7] to archive 2024-11-23T16:16:32,891 DEBUG [StoreCloser-TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-23T16:16:32,892 DEBUG [StoreCloser-TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/A/b1c124a4b0a34192aaaace777e6da0f1 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/A/b1c124a4b0a34192aaaace777e6da0f1 2024-11-23T16:16:32,893 DEBUG [StoreCloser-TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/A/19b2fe1b7c5a42cf9a642083b4edf826 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/A/19b2fe1b7c5a42cf9a642083b4edf826 2024-11-23T16:16:32,894 DEBUG [StoreCloser-TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/A/9032ef42731641baba6d8904f48110c4 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/A/9032ef42731641baba6d8904f48110c4 2024-11-23T16:16:32,895 DEBUG [StoreCloser-TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/A/d1ab2ef581cc419bb126c47d56b4e937 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/A/d1ab2ef581cc419bb126c47d56b4e937 2024-11-23T16:16:32,896 DEBUG [StoreCloser-TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/A/7a7b39e3beb34eb08f2acffbbfa21342 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/A/7a7b39e3beb34eb08f2acffbbfa21342 2024-11-23T16:16:32,897 DEBUG [StoreCloser-TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/A/6d967083128640b8a90cbd86180672b3 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/A/6d967083128640b8a90cbd86180672b3 2024-11-23T16:16:32,898 DEBUG [StoreCloser-TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/A/58f564b66ed54635a5e31adb9aeba85f to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/A/58f564b66ed54635a5e31adb9aeba85f 2024-11-23T16:16:32,899 DEBUG [StoreCloser-TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/A/3f5f1c6d7b4e43e28da90dcb2b10f617 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/A/3f5f1c6d7b4e43e28da90dcb2b10f617 2024-11-23T16:16:32,901 DEBUG [StoreCloser-TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/A/f7c5657eeafc4f5197739834d768e9ba to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/A/f7c5657eeafc4f5197739834d768e9ba 2024-11-23T16:16:32,902 DEBUG [StoreCloser-TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/A/fcea5c7418154dffac3ea561659c88e7 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/A/fcea5c7418154dffac3ea561659c88e7 2024-11-23T16:16:32,903 DEBUG [StoreCloser-TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/A/88d5b8fb2b434b08a7306e7fb591eda5 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/A/88d5b8fb2b434b08a7306e7fb591eda5 2024-11-23T16:16:32,904 DEBUG [StoreCloser-TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/A/1188308ed1a941c3b82febd0c0cf453b to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/A/1188308ed1a941c3b82febd0c0cf453b 2024-11-23T16:16:32,905 DEBUG [StoreCloser-TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/A/46b30f27a7ed401da4d65c59ae93ef22 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/A/46b30f27a7ed401da4d65c59ae93ef22 2024-11-23T16:16:32,907 DEBUG [StoreCloser-TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/A/295a9dd244a24539bc8c0785c393891d to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/A/295a9dd244a24539bc8c0785c393891d 2024-11-23T16:16:32,908 DEBUG [StoreCloser-TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/A/d8e6279db93b436182af0f4d8d6ac5e1 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/A/d8e6279db93b436182af0f4d8d6ac5e1 2024-11-23T16:16:32,908 DEBUG [StoreCloser-TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/A/9a478905b9d04f659c11c62d3feee135 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/A/9a478905b9d04f659c11c62d3feee135 2024-11-23T16:16:32,909 DEBUG [StoreCloser-TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/A/6b31d0cb8b814bf7ace0849cafd2165e to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/A/6b31d0cb8b814bf7ace0849cafd2165e 2024-11-23T16:16:32,910 DEBUG [StoreCloser-TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/A/1ab0b906297a4b9a825c5dc4c1c239f7 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/A/1ab0b906297a4b9a825c5dc4c1c239f7 2024-11-23T16:16:32,912 DEBUG [StoreCloser-TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/B/daa00491345a4740bf7938eed86e2301, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/B/34d1b0a989f14c419717ba7da246b8db, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/B/1c152fa1818f41ada2915546d5afae8c, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/B/26ba178db8a54e50a6d897c63d5aa5ec, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/B/57cd3c9ba67c4bafb7fe46ed0631459c, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/B/e7717a4adce94acca5a666b8d8492954, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/B/c16dc2b9be6a4d4c82455961056d43f8, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/B/ffdeaa2c96f34aedbf00684d36a3bb34, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/B/79af371c8b3343d09fc762b6c3b3b392, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/B/96f7bdc38ca3425995ffeafab1347dff, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/B/dda8b67939ed4d189432c59113213813, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/B/8df8e5399fb443b88368bb162fa41a64, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/B/6a7a82c2177441b3bd67e96df594a850, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/B/fd0343b5c1f14747bc975805f9553146, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/B/f0e40ac2d3564dbe8a3082e76342d115, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/B/1ad4c0bb34fa4001ba0c2b8ad0ffc656, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/B/1cd046d016d54c2dafc646167a1bc893, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/B/76f499efa7464821a3fa94a888af24cd] to archive 2024-11-23T16:16:32,912 DEBUG [StoreCloser-TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-23T16:16:32,913 DEBUG [StoreCloser-TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/B/daa00491345a4740bf7938eed86e2301 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/B/daa00491345a4740bf7938eed86e2301 2024-11-23T16:16:32,914 DEBUG [StoreCloser-TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/B/34d1b0a989f14c419717ba7da246b8db to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/B/34d1b0a989f14c419717ba7da246b8db 2024-11-23T16:16:32,915 DEBUG [StoreCloser-TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/B/1c152fa1818f41ada2915546d5afae8c to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/B/1c152fa1818f41ada2915546d5afae8c 2024-11-23T16:16:32,915 DEBUG [StoreCloser-TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/B/26ba178db8a54e50a6d897c63d5aa5ec to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/B/26ba178db8a54e50a6d897c63d5aa5ec 2024-11-23T16:16:32,916 DEBUG [StoreCloser-TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/B/57cd3c9ba67c4bafb7fe46ed0631459c to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/B/57cd3c9ba67c4bafb7fe46ed0631459c 2024-11-23T16:16:32,917 DEBUG [StoreCloser-TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/B/e7717a4adce94acca5a666b8d8492954 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/B/e7717a4adce94acca5a666b8d8492954 2024-11-23T16:16:32,918 DEBUG [StoreCloser-TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/B/c16dc2b9be6a4d4c82455961056d43f8 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/B/c16dc2b9be6a4d4c82455961056d43f8 2024-11-23T16:16:32,919 DEBUG [StoreCloser-TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/B/ffdeaa2c96f34aedbf00684d36a3bb34 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/B/ffdeaa2c96f34aedbf00684d36a3bb34 2024-11-23T16:16:32,919 DEBUG [StoreCloser-TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/B/79af371c8b3343d09fc762b6c3b3b392 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/B/79af371c8b3343d09fc762b6c3b3b392 2024-11-23T16:16:32,920 DEBUG [StoreCloser-TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/B/96f7bdc38ca3425995ffeafab1347dff to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/B/96f7bdc38ca3425995ffeafab1347dff 2024-11-23T16:16:32,921 DEBUG [StoreCloser-TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/B/dda8b67939ed4d189432c59113213813 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/B/dda8b67939ed4d189432c59113213813 2024-11-23T16:16:32,922 DEBUG [StoreCloser-TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/B/8df8e5399fb443b88368bb162fa41a64 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/B/8df8e5399fb443b88368bb162fa41a64 2024-11-23T16:16:32,923 DEBUG [StoreCloser-TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/B/6a7a82c2177441b3bd67e96df594a850 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/B/6a7a82c2177441b3bd67e96df594a850 2024-11-23T16:16:32,924 DEBUG [StoreCloser-TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/B/fd0343b5c1f14747bc975805f9553146 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/B/fd0343b5c1f14747bc975805f9553146 2024-11-23T16:16:32,925 DEBUG [StoreCloser-TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/B/f0e40ac2d3564dbe8a3082e76342d115 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/B/f0e40ac2d3564dbe8a3082e76342d115 2024-11-23T16:16:32,926 DEBUG [StoreCloser-TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/B/1ad4c0bb34fa4001ba0c2b8ad0ffc656 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/B/1ad4c0bb34fa4001ba0c2b8ad0ffc656 2024-11-23T16:16:32,927 DEBUG [StoreCloser-TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/B/1cd046d016d54c2dafc646167a1bc893 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/B/1cd046d016d54c2dafc646167a1bc893 2024-11-23T16:16:32,927 DEBUG [StoreCloser-TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/B/76f499efa7464821a3fa94a888af24cd to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/B/76f499efa7464821a3fa94a888af24cd 2024-11-23T16:16:32,929 DEBUG [StoreCloser-TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/C/d821084003594d5fae0df646a986689c, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/C/66b396aa48af4c668f52e8c6f3905c1b, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/C/18f634731ea24c58b1b2b3c86fcb1bf6, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/C/d8b90a5980094068899bf16be1cf178b, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/C/127137f654b54bad94ee1fcec6c22531, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/C/83f7f479ed3c468899cbf3528bd681d2, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/C/cbca53d9e84b48a3ab7a5145274feb17, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/C/302a6aeb45de4b8f8365f3206713d10c, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/C/891b5a1cb3fb4bab82596b5bfe06ccfb, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/C/7e9f3e3dd9354c3da45c652b735bacaf, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/C/654750eb4f1b4b5494ab2e6869513f84, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/C/7672978d77c44e29a7fd94b901538305, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/C/68d8914b3afd4f6fbbfbb65d84a1628f, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/C/2164692c3ca6456587a245be04e673fa, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/C/3f0c45c085b24995861cacaedfaba9de, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/C/d5f630abe14c4ed790930c2c46a9c90b, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/C/64627dab0b494ba0be74bfaf9ce0a944, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/C/9fe451c28dbe4cba892282eefb819879] to archive 2024-11-23T16:16:32,929 DEBUG [StoreCloser-TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-23T16:16:32,930 DEBUG [StoreCloser-TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/C/d821084003594d5fae0df646a986689c to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/C/d821084003594d5fae0df646a986689c 2024-11-23T16:16:32,931 DEBUG [StoreCloser-TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/C/66b396aa48af4c668f52e8c6f3905c1b to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/C/66b396aa48af4c668f52e8c6f3905c1b 2024-11-23T16:16:32,932 DEBUG [StoreCloser-TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/C/18f634731ea24c58b1b2b3c86fcb1bf6 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/C/18f634731ea24c58b1b2b3c86fcb1bf6 2024-11-23T16:16:32,933 DEBUG [StoreCloser-TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/C/d8b90a5980094068899bf16be1cf178b to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/C/d8b90a5980094068899bf16be1cf178b 2024-11-23T16:16:32,934 DEBUG [StoreCloser-TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/C/127137f654b54bad94ee1fcec6c22531 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/C/127137f654b54bad94ee1fcec6c22531 2024-11-23T16:16:32,935 DEBUG [StoreCloser-TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/C/83f7f479ed3c468899cbf3528bd681d2 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/C/83f7f479ed3c468899cbf3528bd681d2 2024-11-23T16:16:32,935 DEBUG [StoreCloser-TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/C/cbca53d9e84b48a3ab7a5145274feb17 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/C/cbca53d9e84b48a3ab7a5145274feb17 2024-11-23T16:16:32,936 DEBUG [StoreCloser-TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/C/302a6aeb45de4b8f8365f3206713d10c to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/C/302a6aeb45de4b8f8365f3206713d10c 2024-11-23T16:16:32,937 DEBUG [StoreCloser-TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/C/891b5a1cb3fb4bab82596b5bfe06ccfb to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/C/891b5a1cb3fb4bab82596b5bfe06ccfb 2024-11-23T16:16:32,938 DEBUG [StoreCloser-TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/C/7e9f3e3dd9354c3da45c652b735bacaf to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/C/7e9f3e3dd9354c3da45c652b735bacaf 2024-11-23T16:16:32,939 DEBUG [StoreCloser-TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/C/654750eb4f1b4b5494ab2e6869513f84 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/C/654750eb4f1b4b5494ab2e6869513f84 2024-11-23T16:16:32,940 DEBUG [StoreCloser-TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/C/7672978d77c44e29a7fd94b901538305 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/C/7672978d77c44e29a7fd94b901538305 2024-11-23T16:16:32,941 DEBUG [StoreCloser-TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/C/68d8914b3afd4f6fbbfbb65d84a1628f to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/C/68d8914b3afd4f6fbbfbb65d84a1628f 2024-11-23T16:16:32,941 DEBUG [StoreCloser-TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/C/2164692c3ca6456587a245be04e673fa to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/C/2164692c3ca6456587a245be04e673fa 2024-11-23T16:16:32,942 DEBUG [StoreCloser-TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/C/3f0c45c085b24995861cacaedfaba9de to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/C/3f0c45c085b24995861cacaedfaba9de 2024-11-23T16:16:32,943 DEBUG [StoreCloser-TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/C/d5f630abe14c4ed790930c2c46a9c90b to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/C/d5f630abe14c4ed790930c2c46a9c90b 2024-11-23T16:16:32,944 DEBUG [StoreCloser-TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/C/64627dab0b494ba0be74bfaf9ce0a944 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/C/64627dab0b494ba0be74bfaf9ce0a944 2024-11-23T16:16:32,945 DEBUG [StoreCloser-TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/C/9fe451c28dbe4cba892282eefb819879 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/C/9fe451c28dbe4cba892282eefb819879 2024-11-23T16:16:32,949 DEBUG [RS_CLOSE_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_CLOSE_REGION, pid=157}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/recovered.edits/286.seqid, newMaxSeqId=286, maxSeqId=4 2024-11-23T16:16:32,950 INFO [RS_CLOSE_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_CLOSE_REGION, pid=157}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36. 2024-11-23T16:16:32,950 DEBUG [RS_CLOSE_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_CLOSE_REGION, pid=157}] regionserver.HRegion(1676): Region close journal for 1930901e64343eeb48a7930647197c36: Waiting for close lock at 1732378592020Running coprocessor pre-close hooks at 1732378592020Disabling compacts and flushes for region at 1732378592020Disabling writes for close at 1732378592020Obtaining lock to block concurrent updates at 1732378592020Preparing flush snapshotting stores in 1930901e64343eeb48a7930647197c36 at 1732378592020Finished memstore snapshotting TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1732378592021 (+1 ms)Flushing stores of TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36. at 1732378592021Flushing 1930901e64343eeb48a7930647197c36/A: creating writer at 1732378592021Flushing mob file 1930901e64343eeb48a7930647197c36/A: appending metadata at 1732378592027 (+6 ms)Flushing mob file 1930901e64343eeb48a7930647197c36/A: closing flushed file at 1732378592027Flushing 1930901e64343eeb48a7930647197c36/A: appending metadata at 1732378592043 (+16 ms)Flushing 1930901e64343eeb48a7930647197c36/A: closing flushed file at 1732378592044 (+1 ms)Flushing 1930901e64343eeb48a7930647197c36/B: creating writer at 1732378592055 (+11 ms)Flushing 1930901e64343eeb48a7930647197c36/B: appending metadata at 1732378592057 (+2 ms)Flushing 1930901e64343eeb48a7930647197c36/B: closing flushed file at 1732378592057Flushing 1930901e64343eeb48a7930647197c36/C: creating writer at 1732378592465 (+408 ms)Flushing 1930901e64343eeb48a7930647197c36/C: appending metadata at 1732378592467 (+2 ms)Flushing 1930901e64343eeb48a7930647197c36/C: closing flushed file at 1732378592467Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@12092fc4: reopening flushed file at 1732378592873 (+406 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5bbc0311: reopening flushed file at 1732378592881 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@215ea41b: reopening flushed file at 1732378592885 (+4 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 1930901e64343eeb48a7930647197c36 in 870ms, sequenceid=283, compaction requested=true at 1732378592890 (+5 ms)Writing region close event to WAL at 1732378592946 (+56 ms)Running coprocessor post-close hooks at 1732378592950 (+4 ms)Closed at 1732378592950 2024-11-23T16:16:32,952 INFO [RS_CLOSE_REGION-regionserver/1a7c5470b472:0-0 {event_type=M_RS_CLOSE_REGION, pid=157}] handler.UnassignRegionHandler(157): Closed 1930901e64343eeb48a7930647197c36 2024-11-23T16:16:32,952 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=156 updating hbase:meta row=1930901e64343eeb48a7930647197c36, regionState=CLOSED 2024-11-23T16:16:32,954 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=157, ppid=156, state=RUNNABLE, hasLock=false; CloseRegionProcedure 1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 because future has completed 2024-11-23T16:16:32,956 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=157, resume processing ppid=156 2024-11-23T16:16:32,956 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=157, ppid=156, state=SUCCESS, hasLock=false; CloseRegionProcedure 1930901e64343eeb48a7930647197c36, server=1a7c5470b472,44715,1732378420375 in 1.0860 sec 2024-11-23T16:16:32,957 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=156, resume processing ppid=155 2024-11-23T16:16:32,957 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=156, ppid=155, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1930901e64343eeb48a7930647197c36, UNASSIGN in 1.0910 sec 2024-11-23T16:16:32,959 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=155, resume processing ppid=154 2024-11-23T16:16:32,959 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=155, ppid=154, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.0940 sec 2024-11-23T16:16:32,960 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732378592960"}]},"ts":"1732378592960"} 2024-11-23T16:16:32,961 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-23T16:16:32,961 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-23T16:16:32,963 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=154, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.1030 sec 2024-11-23T16:16:32,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-23T16:16:32,998 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-23T16:16:32,999 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45799 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-23T16:16:33,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45799 {}] procedure2.ProcedureExecutor(1139): Stored pid=158, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-23T16:16:33,000 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=158, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-23T16:16:33,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-23T16:16:33,001 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=158, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-23T16:16:33,003 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36 2024-11-23T16:16:33,005 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/A, FileablePath, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/B, FileablePath, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/C, FileablePath, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/recovered.edits] 2024-11-23T16:16:33,007 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/A/6c0b5249fc794e639df6f9257b284b33 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/A/6c0b5249fc794e639df6f9257b284b33 2024-11-23T16:16:33,009 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/A/81fcc72f78b9437c8b1f0740beb6c2f0 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/A/81fcc72f78b9437c8b1f0740beb6c2f0 2024-11-23T16:16:33,010 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/A/9ee67db43b4a48c4ac0843613e404127 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/A/9ee67db43b4a48c4ac0843613e404127 2024-11-23T16:16:33,011 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/B/01de1b3aba2b4eac875844ab5f24c5c4 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/B/01de1b3aba2b4eac875844ab5f24c5c4 2024-11-23T16:16:33,012 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/B/6ed9e8bd0e3f4d63a13da6131f60c1fb to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/B/6ed9e8bd0e3f4d63a13da6131f60c1fb 2024-11-23T16:16:33,013 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/B/fab1902dcf84438c9a0cda1efb79744e to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/B/fab1902dcf84438c9a0cda1efb79744e 2024-11-23T16:16:33,014 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/C/457eb4fcb859406d89c66337ce2fb7d6 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/C/457eb4fcb859406d89c66337ce2fb7d6 2024-11-23T16:16:33,015 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/C/4d49dafa561d447d8398fab71bfee5a6 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/C/4d49dafa561d447d8398fab71bfee5a6 2024-11-23T16:16:33,016 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/C/5cb134a3dbd049c79551f9a19b0489db to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/C/5cb134a3dbd049c79551f9a19b0489db 2024-11-23T16:16:33,018 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/recovered.edits/286.seqid to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36/recovered.edits/286.seqid 2024-11-23T16:16:33,018 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/default/TestAcidGuarantees/1930901e64343eeb48a7930647197c36 2024-11-23T16:16:33,019 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-23T16:16:33,019 DEBUG [PEWorker-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-23T16:16:33,020 DEBUG [PEWorker-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-23T16:16:33,022 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112308bab72003394a44b9e7bcbb682d6a07_1930901e64343eeb48a7930647197c36 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112308bab72003394a44b9e7bcbb682d6a07_1930901e64343eeb48a7930647197c36 2024-11-23T16:16:33,023 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411230b578efdb7b44b039bf6833dd0e0e78c_1930901e64343eeb48a7930647197c36 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411230b578efdb7b44b039bf6833dd0e0e78c_1930901e64343eeb48a7930647197c36 2024-11-23T16:16:33,024 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112310273c7bb5714d088022d2f4768964bc_1930901e64343eeb48a7930647197c36 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112310273c7bb5714d088022d2f4768964bc_1930901e64343eeb48a7930647197c36 2024-11-23T16:16:33,025 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411231dbfaf115d994a3a97dc6ddcfb418b7c_1930901e64343eeb48a7930647197c36 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411231dbfaf115d994a3a97dc6ddcfb418b7c_1930901e64343eeb48a7930647197c36 2024-11-23T16:16:33,026 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411232a6d625c2c82481da260d7a45c8fddbb_1930901e64343eeb48a7930647197c36 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411232a6d625c2c82481da260d7a45c8fddbb_1930901e64343eeb48a7930647197c36 2024-11-23T16:16:33,027 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411232d88340642604ed19002eafb984debe0_1930901e64343eeb48a7930647197c36 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411232d88340642604ed19002eafb984debe0_1930901e64343eeb48a7930647197c36 2024-11-23T16:16:33,028 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411234827fee1747f446cb0c20bac3a531005_1930901e64343eeb48a7930647197c36 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411234827fee1747f446cb0c20bac3a531005_1930901e64343eeb48a7930647197c36 2024-11-23T16:16:33,029 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411235fbe8cf7f8a14ea585a13e4c516cf298_1930901e64343eeb48a7930647197c36 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411235fbe8cf7f8a14ea585a13e4c516cf298_1930901e64343eeb48a7930647197c36 2024-11-23T16:16:33,030 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411235ff4927dbec8486c94f4017980f1a48f_1930901e64343eeb48a7930647197c36 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411235ff4927dbec8486c94f4017980f1a48f_1930901e64343eeb48a7930647197c36 2024-11-23T16:16:33,031 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112361c766f75ca84e30b73aa163d0a544a7_1930901e64343eeb48a7930647197c36 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112361c766f75ca84e30b73aa163d0a544a7_1930901e64343eeb48a7930647197c36 2024-11-23T16:16:33,032 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411237c41755203e3479d89f7010f7d0895fa_1930901e64343eeb48a7930647197c36 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411237c41755203e3479d89f7010f7d0895fa_1930901e64343eeb48a7930647197c36 2024-11-23T16:16:33,033 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123ded1c4ea8aaa44dd9db45222062d413f_1930901e64343eeb48a7930647197c36 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123ded1c4ea8aaa44dd9db45222062d413f_1930901e64343eeb48a7930647197c36 2024-11-23T16:16:33,034 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123e6d7fa9be6a24ce185dde45fc0d96dba_1930901e64343eeb48a7930647197c36 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123e6d7fa9be6a24ce185dde45fc0d96dba_1930901e64343eeb48a7930647197c36 2024-11-23T16:16:33,035 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123ec7eaebc3695467ca34d4f5a9de8945c_1930901e64343eeb48a7930647197c36 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123ec7eaebc3695467ca34d4f5a9de8945c_1930901e64343eeb48a7930647197c36 2024-11-23T16:16:33,036 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123ff1b460233594db69b5d5dbbf38ca8bd_1930901e64343eeb48a7930647197c36 to hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123ff1b460233594db69b5d5dbbf38ca8bd_1930901e64343eeb48a7930647197c36 2024-11-23T16:16:33,036 DEBUG [PEWorker-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-23T16:16:33,038 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=158, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-23T16:16:33,040 WARN [PEWorker-2 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-23T16:16:33,042 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-23T16:16:33,043 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=158, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-23T16:16:33,043 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-23T16:16:33,043 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732378593043"}]},"ts":"9223372036854775807"} 2024-11-23T16:16:33,050 INFO [PEWorker-2 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-23T16:16:33,050 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 1930901e64343eeb48a7930647197c36, NAME => 'TestAcidGuarantees,,1732378565636.1930901e64343eeb48a7930647197c36.', STARTKEY => '', ENDKEY => ''}] 2024-11-23T16:16:33,050 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-23T16:16:33,051 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732378593050"}]},"ts":"9223372036854775807"} 2024-11-23T16:16:33,052 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-23T16:16:33,053 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(133): Finished pid=158, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-23T16:16:33,055 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=158, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 54 msec 2024-11-23T16:16:33,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-23T16:16:33,108 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-23T16:16:33,109 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-23T16:16:33,119 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobGetAtomicity Thread=252 (was 250) - Thread LEAK? -, OpenFileDescriptor=455 (was 449) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=394 (was 395), ProcessCount=11 (was 11), AvailableMemoryMB=6684 (was 6734) 2024-11-23T16:16:33,119 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-23T16:16:33,119 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-23T16:16:33,119 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-23T16:16:33,119 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T16:16:33,119 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T16:16:33,119 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-23T16:16:33,119 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-23T16:16:33,119 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=2127823695, stopped=false 2024-11-23T16:16:33,120 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=1a7c5470b472,45799,1732378419617 2024-11-23T16:16:33,122 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45799-0x101459ac8920000, quorum=127.0.0.1:59338, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-23T16:16:33,122 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45799-0x101459ac8920000, quorum=127.0.0.1:59338, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T16:16:33,122 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44715-0x101459ac8920001, quorum=127.0.0.1:59338, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-23T16:16:33,122 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-23T16:16:33,122 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44715-0x101459ac8920001, quorum=127.0.0.1:59338, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T16:16:33,122 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-23T16:16:33,123 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-23T16:16:33,123 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T16:16:33,123 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:44715-0x101459ac8920001, quorum=127.0.0.1:59338, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-23T16:16:33,123 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:45799-0x101459ac8920000, quorum=127.0.0.1:59338, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-23T16:16:33,123 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '1a7c5470b472,44715,1732378420375' ***** 2024-11-23T16:16:33,123 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-23T16:16:33,124 INFO [RS:0;1a7c5470b472:44715 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-23T16:16:33,124 INFO [RS:0;1a7c5470b472:44715 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-23T16:16:33,124 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-23T16:16:33,124 INFO [RS:0;1a7c5470b472:44715 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-23T16:16:33,124 INFO [RS:0;1a7c5470b472:44715 {}] regionserver.HRegionServer(959): stopping server 1a7c5470b472,44715,1732378420375 2024-11-23T16:16:33,124 INFO [RS:0;1a7c5470b472:44715 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-23T16:16:33,124 INFO [RS:0;1a7c5470b472:44715 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;1a7c5470b472:44715. 2024-11-23T16:16:33,125 DEBUG [RS:0;1a7c5470b472:44715 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-23T16:16:33,125 DEBUG [RS:0;1a7c5470b472:44715 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T16:16:33,125 INFO [RS:0;1a7c5470b472:44715 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-23T16:16:33,125 INFO [RS:0;1a7c5470b472:44715 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-23T16:16:33,125 INFO [RS:0;1a7c5470b472:44715 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-23T16:16:33,125 INFO [RS:0;1a7c5470b472:44715 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-23T16:16:33,126 INFO [RS:0;1a7c5470b472:44715 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-23T16:16:33,126 DEBUG [RS:0;1a7c5470b472:44715 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-11-23T16:16:33,126 DEBUG [RS:0;1a7c5470b472:44715 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-23T16:16:33,126 DEBUG [RS_CLOSE_META-regionserver/1a7c5470b472:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-23T16:16:33,126 INFO [RS_CLOSE_META-regionserver/1a7c5470b472:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-23T16:16:33,126 DEBUG [RS_CLOSE_META-regionserver/1a7c5470b472:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-23T16:16:33,127 DEBUG [RS_CLOSE_META-regionserver/1a7c5470b472:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-23T16:16:33,127 DEBUG [RS_CLOSE_META-regionserver/1a7c5470b472:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-23T16:16:33,127 INFO [RS_CLOSE_META-regionserver/1a7c5470b472:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=19.91 KB heapSize=35.20 KB 2024-11-23T16:16:33,151 DEBUG [RS_CLOSE_META-regionserver/1a7c5470b472:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/hbase/meta/1588230740/.tmp/info/368440d0a8434ae3881cae4f551036ac is 95, key is TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302./info:/1732378449879/DeleteFamily/seqid=0 2024-11-23T16:16:33,155 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742423_1599 (size=6514) 2024-11-23T16:16:33,156 INFO [regionserver/1a7c5470b472:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-23T16:16:33,327 DEBUG [RS:0;1a7c5470b472:44715 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-23T16:16:33,527 DEBUG [RS:0;1a7c5470b472:44715 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-23T16:16:33,556 INFO [RS_CLOSE_META-regionserver/1a7c5470b472:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=16.75 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/hbase/meta/1588230740/.tmp/info/368440d0a8434ae3881cae4f551036ac 2024-11-23T16:16:33,578 DEBUG [RS_CLOSE_META-regionserver/1a7c5470b472:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/hbase/meta/1588230740/.tmp/ns/871591c0faee4cc39b5a2bf7a0a4f42d is 93, key is TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302./ns:/1732378449829/DeleteFamily/seqid=0 2024-11-23T16:16:33,582 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742424_1600 (size=5906) 2024-11-23T16:16:33,582 INFO [RS_CLOSE_META-regionserver/1a7c5470b472:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=608 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/hbase/meta/1588230740/.tmp/ns/871591c0faee4cc39b5a2bf7a0a4f42d 2024-11-23T16:16:33,602 DEBUG [RS_CLOSE_META-regionserver/1a7c5470b472:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/hbase/meta/1588230740/.tmp/rep_barrier/07d38fcae34147898fc2c7e5d8fc0d34 is 102, key is TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302./rep_barrier:/1732378449829/DeleteFamily/seqid=0 2024-11-23T16:16:33,605 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742425_1601 (size=6025) 2024-11-23T16:16:33,727 DEBUG [RS:0;1a7c5470b472:44715 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-23T16:16:33,889 INFO [regionserver/1a7c5470b472:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-23T16:16:33,890 INFO [regionserver/1a7c5470b472:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-23T16:16:33,927 DEBUG [RS:0;1a7c5470b472:44715 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-23T16:16:34,006 INFO [RS_CLOSE_META-regionserver/1a7c5470b472:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=588 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/hbase/meta/1588230740/.tmp/rep_barrier/07d38fcae34147898fc2c7e5d8fc0d34 2024-11-23T16:16:34,024 DEBUG [RS_CLOSE_META-regionserver/1a7c5470b472:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/hbase/meta/1588230740/.tmp/table/1d5cf14ed55040eb84c48390f2268837 is 96, key is TestAcidGuarantees,,1732378424137.4550087eb34ce466d194f9ce96539302./table:/1732378449829/DeleteFamily/seqid=0 2024-11-23T16:16:34,027 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742426_1602 (size=5936) 2024-11-23T16:16:34,127 DEBUG [RS:0;1a7c5470b472:44715 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-23T16:16:34,328 DEBUG [RS:0;1a7c5470b472:44715 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-23T16:16:34,428 INFO [RS_CLOSE_META-regionserver/1a7c5470b472:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.99 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/hbase/meta/1588230740/.tmp/table/1d5cf14ed55040eb84c48390f2268837 2024-11-23T16:16:34,431 DEBUG [RS_CLOSE_META-regionserver/1a7c5470b472:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/hbase/meta/1588230740/.tmp/info/368440d0a8434ae3881cae4f551036ac as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/hbase/meta/1588230740/info/368440d0a8434ae3881cae4f551036ac 2024-11-23T16:16:34,434 INFO [RS_CLOSE_META-regionserver/1a7c5470b472:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/hbase/meta/1588230740/info/368440d0a8434ae3881cae4f551036ac, entries=12, sequenceid=90, filesize=6.4 K 2024-11-23T16:16:34,435 DEBUG [RS_CLOSE_META-regionserver/1a7c5470b472:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/hbase/meta/1588230740/.tmp/ns/871591c0faee4cc39b5a2bf7a0a4f42d as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/hbase/meta/1588230740/ns/871591c0faee4cc39b5a2bf7a0a4f42d 2024-11-23T16:16:34,437 INFO [RS_CLOSE_META-regionserver/1a7c5470b472:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/hbase/meta/1588230740/ns/871591c0faee4cc39b5a2bf7a0a4f42d, entries=8, sequenceid=90, filesize=5.8 K 2024-11-23T16:16:34,438 DEBUG [RS_CLOSE_META-regionserver/1a7c5470b472:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/hbase/meta/1588230740/.tmp/rep_barrier/07d38fcae34147898fc2c7e5d8fc0d34 as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/hbase/meta/1588230740/rep_barrier/07d38fcae34147898fc2c7e5d8fc0d34 2024-11-23T16:16:34,441 INFO [RS_CLOSE_META-regionserver/1a7c5470b472:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/hbase/meta/1588230740/rep_barrier/07d38fcae34147898fc2c7e5d8fc0d34, entries=6, sequenceid=90, filesize=5.9 K 2024-11-23T16:16:34,441 DEBUG [RS_CLOSE_META-regionserver/1a7c5470b472:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/hbase/meta/1588230740/.tmp/table/1d5cf14ed55040eb84c48390f2268837 as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/hbase/meta/1588230740/table/1d5cf14ed55040eb84c48390f2268837 2024-11-23T16:16:34,444 INFO [RS_CLOSE_META-regionserver/1a7c5470b472:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/hbase/meta/1588230740/table/1d5cf14ed55040eb84c48390f2268837, entries=7, sequenceid=90, filesize=5.8 K 2024-11-23T16:16:34,445 INFO [RS_CLOSE_META-regionserver/1a7c5470b472:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1317ms, sequenceid=90, compaction requested=false 2024-11-23T16:16:34,450 DEBUG [RS_CLOSE_META-regionserver/1a7c5470b472:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/data/hbase/meta/1588230740/recovered.edits/93.seqid, newMaxSeqId=93, maxSeqId=1 2024-11-23T16:16:34,451 DEBUG [RS_CLOSE_META-regionserver/1a7c5470b472:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-23T16:16:34,451 INFO [RS_CLOSE_META-regionserver/1a7c5470b472:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-23T16:16:34,451 DEBUG [RS_CLOSE_META-regionserver/1a7c5470b472:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732378593126Running coprocessor pre-close hooks at 1732378593126Disabling compacts and flushes for region at 1732378593126Disabling writes for close at 1732378593127 (+1 ms)Obtaining lock to block concurrent updates at 1732378593127Preparing flush snapshotting stores in 1588230740 at 1732378593127Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=20384, getHeapSize=35984, getOffHeapSize=0, getCellsCount=191 at 1732378593127Flushing stores of hbase:meta,,1.1588230740 at 1732378593128 (+1 ms)Flushing 1588230740/info: creating writer at 1732378593128Flushing 1588230740/info: appending metadata at 1732378593151 (+23 ms)Flushing 1588230740/info: closing flushed file at 1732378593151Flushing 1588230740/ns: creating writer at 1732378593560 (+409 ms)Flushing 1588230740/ns: appending metadata at 1732378593578 (+18 ms)Flushing 1588230740/ns: closing flushed file at 1732378593578Flushing 1588230740/rep_barrier: creating writer at 1732378593586 (+8 ms)Flushing 1588230740/rep_barrier: appending metadata at 1732378593602 (+16 ms)Flushing 1588230740/rep_barrier: closing flushed file at 1732378593602Flushing 1588230740/table: creating writer at 1732378594009 (+407 ms)Flushing 1588230740/table: appending metadata at 1732378594023 (+14 ms)Flushing 1588230740/table: closing flushed file at 1732378594023Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1c5612d3: reopening flushed file at 1732378594431 (+408 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@57ea150c: reopening flushed file at 1732378594434 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@574c092: reopening flushed file at 1732378594437 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@64a8c497: reopening flushed file at 1732378594441 (+4 ms)Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1317ms, sequenceid=90, compaction requested=false at 1732378594445 (+4 ms)Writing region close event to WAL at 1732378594448 (+3 ms)Running coprocessor post-close hooks at 1732378594451 (+3 ms)Closed at 1732378594451 2024-11-23T16:16:34,451 DEBUG [RS_CLOSE_META-regionserver/1a7c5470b472:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-23T16:16:34,528 INFO [RS:0;1a7c5470b472:44715 {}] regionserver.HRegionServer(976): stopping server 1a7c5470b472,44715,1732378420375; all regions closed. 2024-11-23T16:16:34,532 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073741834_1010 (size=25344) 2024-11-23T16:16:34,534 DEBUG [RS:0;1a7c5470b472:44715 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/oldWALs 2024-11-23T16:16:34,534 INFO [RS:0;1a7c5470b472:44715 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 1a7c5470b472%2C44715%2C1732378420375.meta:.meta(num 1732378423567) 2024-11-23T16:16:34,536 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073741833_1009 (size=14532933) 2024-11-23T16:16:34,537 DEBUG [RS:0;1a7c5470b472:44715 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/oldWALs 2024-11-23T16:16:34,537 INFO [RS:0;1a7c5470b472:44715 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 1a7c5470b472%2C44715%2C1732378420375:(num 1732378423053) 2024-11-23T16:16:34,537 DEBUG [RS:0;1a7c5470b472:44715 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T16:16:34,537 INFO [RS:0;1a7c5470b472:44715 {}] regionserver.LeaseManager(133): Closed leases 2024-11-23T16:16:34,538 INFO [RS:0;1a7c5470b472:44715 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-23T16:16:34,538 INFO [RS:0;1a7c5470b472:44715 {}] hbase.ChoreService(370): Chore service for: regionserver/1a7c5470b472:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-23T16:16:34,538 INFO [RS:0;1a7c5470b472:44715 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-23T16:16:34,538 INFO [regionserver/1a7c5470b472:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-23T16:16:34,539 INFO [RS:0;1a7c5470b472:44715 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:44715 2024-11-23T16:16:34,544 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44715-0x101459ac8920001, quorum=127.0.0.1:59338, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/1a7c5470b472,44715,1732378420375 2024-11-23T16:16:34,544 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45799-0x101459ac8920000, quorum=127.0.0.1:59338, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-23T16:16:34,544 INFO [RS:0;1a7c5470b472:44715 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-23T16:16:34,546 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [1a7c5470b472,44715,1732378420375] 2024-11-23T16:16:34,548 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/1a7c5470b472,44715,1732378420375 already deleted, retry=false 2024-11-23T16:16:34,548 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 1a7c5470b472,44715,1732378420375 expired; onlineServers=0 2024-11-23T16:16:34,548 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '1a7c5470b472,45799,1732378419617' ***** 2024-11-23T16:16:34,548 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-23T16:16:34,548 INFO [M:0;1a7c5470b472:45799 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-23T16:16:34,548 INFO [M:0;1a7c5470b472:45799 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-23T16:16:34,548 DEBUG [M:0;1a7c5470b472:45799 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-23T16:16:34,548 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-23T16:16:34,548 DEBUG [M:0;1a7c5470b472:45799 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-23T16:16:34,548 DEBUG [master/1a7c5470b472:0:becomeActiveMaster-HFileCleaner.large.0-1732378422687 {}] cleaner.HFileCleaner(306): Exit Thread[master/1a7c5470b472:0:becomeActiveMaster-HFileCleaner.large.0-1732378422687,5,FailOnTimeoutGroup] 2024-11-23T16:16:34,548 DEBUG [master/1a7c5470b472:0:becomeActiveMaster-HFileCleaner.small.0-1732378422688 {}] cleaner.HFileCleaner(306): Exit Thread[master/1a7c5470b472:0:becomeActiveMaster-HFileCleaner.small.0-1732378422688,5,FailOnTimeoutGroup] 2024-11-23T16:16:34,548 INFO [M:0;1a7c5470b472:45799 {}] hbase.ChoreService(370): Chore service for: master/1a7c5470b472:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-23T16:16:34,549 INFO [M:0;1a7c5470b472:45799 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-23T16:16:34,549 DEBUG [M:0;1a7c5470b472:45799 {}] master.HMaster(1795): Stopping service threads 2024-11-23T16:16:34,549 INFO [M:0;1a7c5470b472:45799 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-23T16:16:34,549 INFO [M:0;1a7c5470b472:45799 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-23T16:16:34,549 ERROR [M:0;1a7c5470b472:45799 {}] procedure2.ProcedureExecutor(763): There are still active thread in group java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10], see STDOUT java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] Thread[HFileArchiver-5,5,PEWorkerGroup] Thread[IPC Client (1102635514) connection to localhost/127.0.0.1:39667 from jenkins,5,PEWorkerGroup] Thread[IPC Parameter Sending Thread for localhost/127.0.0.1:39667,5,PEWorkerGroup] Thread[HFileArchiver-6,5,PEWorkerGroup] 2024-11-23T16:16:34,549 INFO [M:0;1a7c5470b472:45799 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-23T16:16:34,550 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-23T16:16:34,550 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45799-0x101459ac8920000, quorum=127.0.0.1:59338, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-23T16:16:34,550 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45799-0x101459ac8920000, quorum=127.0.0.1:59338, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T16:16:34,550 DEBUG [M:0;1a7c5470b472:45799 {}] zookeeper.ZKUtil(347): master:45799-0x101459ac8920000, quorum=127.0.0.1:59338, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-23T16:16:34,550 WARN [M:0;1a7c5470b472:45799 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-23T16:16:34,551 INFO [M:0;1a7c5470b472:45799 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/.lastflushedseqids 2024-11-23T16:16:34,560 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742427_1603 (size=72) 2024-11-23T16:16:34,560 INFO [M:0;1a7c5470b472:45799 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-23T16:16:34,560 INFO [M:0;1a7c5470b472:45799 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-23T16:16:34,560 DEBUG [M:0;1a7c5470b472:45799 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-23T16:16:34,560 INFO [M:0;1a7c5470b472:45799 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-23T16:16:34,560 DEBUG [M:0;1a7c5470b472:45799 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-23T16:16:34,560 DEBUG [M:0;1a7c5470b472:45799 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-23T16:16:34,560 DEBUG [M:0;1a7c5470b472:45799 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-23T16:16:34,560 INFO [M:0;1a7c5470b472:45799 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=743.03 KB heapSize=907.73 KB 2024-11-23T16:16:34,575 DEBUG [M:0;1a7c5470b472:45799 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/dc7fd9da5a1049b4a9374df418188453 is 82, key is hbase:meta,,1/info:regioninfo/1732378423711/Put/seqid=0 2024-11-23T16:16:34,578 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742428_1604 (size=5672) 2024-11-23T16:16:34,647 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44715-0x101459ac8920001, quorum=127.0.0.1:59338, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-23T16:16:34,647 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44715-0x101459ac8920001, quorum=127.0.0.1:59338, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-23T16:16:34,647 INFO [RS:0;1a7c5470b472:44715 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-23T16:16:34,648 INFO [RS:0;1a7c5470b472:44715 {}] regionserver.HRegionServer(1031): Exiting; stopping=1a7c5470b472,44715,1732378420375; zookeeper connection closed. 2024-11-23T16:16:34,648 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@6ac98d49 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@6ac98d49 2024-11-23T16:16:34,649 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-11-23T16:16:34,979 INFO [M:0;1a7c5470b472:45799 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=2031 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/dc7fd9da5a1049b4a9374df418188453 2024-11-23T16:16:34,999 DEBUG [M:0;1a7c5470b472:45799 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/21f935b50b74458e9aad3ab42b9a2d6f is 2481, key is \x00\x00\x00\x00\x00\x00\x00\x87/proc:d/1732378568704/Put/seqid=0 2024-11-23T16:16:35,002 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742429_1605 (size=43332) 2024-11-23T16:16:35,403 INFO [M:0;1a7c5470b472:45799 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=742.47 KB at sequenceid=2031 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/21f935b50b74458e9aad3ab42b9a2d6f 2024-11-23T16:16:35,406 INFO [M:0;1a7c5470b472:45799 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 21f935b50b74458e9aad3ab42b9a2d6f 2024-11-23T16:16:35,421 DEBUG [M:0;1a7c5470b472:45799 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/f73af577b75a4adcba04ea41e746a64b is 69, key is 1a7c5470b472,44715,1732378420375/rs:state/1732378422818/Put/seqid=0 2024-11-23T16:16:35,424 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073742430_1606 (size=5156) 2024-11-23T16:16:35,825 INFO [M:0;1a7c5470b472:45799 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=2031 (bloomFilter=true), to=hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/f73af577b75a4adcba04ea41e746a64b 2024-11-23T16:16:35,828 DEBUG [M:0;1a7c5470b472:45799 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/dc7fd9da5a1049b4a9374df418188453 as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/dc7fd9da5a1049b4a9374df418188453 2024-11-23T16:16:35,831 INFO [M:0;1a7c5470b472:45799 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/dc7fd9da5a1049b4a9374df418188453, entries=8, sequenceid=2031, filesize=5.5 K 2024-11-23T16:16:35,831 DEBUG [M:0;1a7c5470b472:45799 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/21f935b50b74458e9aad3ab42b9a2d6f as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/21f935b50b74458e9aad3ab42b9a2d6f 2024-11-23T16:16:35,833 INFO [M:0;1a7c5470b472:45799 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 21f935b50b74458e9aad3ab42b9a2d6f 2024-11-23T16:16:35,833 INFO [M:0;1a7c5470b472:45799 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/21f935b50b74458e9aad3ab42b9a2d6f, entries=158, sequenceid=2031, filesize=42.3 K 2024-11-23T16:16:35,834 DEBUG [M:0;1a7c5470b472:45799 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/f73af577b75a4adcba04ea41e746a64b as hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/f73af577b75a4adcba04ea41e746a64b 2024-11-23T16:16:35,836 INFO [M:0;1a7c5470b472:45799 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39667/user/jenkins/test-data/2677d8cd-848a-a5cd-4d77-b819d3d56bca/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/f73af577b75a4adcba04ea41e746a64b, entries=1, sequenceid=2031, filesize=5.0 K 2024-11-23T16:16:35,837 INFO [M:0;1a7c5470b472:45799 {}] regionserver.HRegion(3140): Finished flush of dataSize ~743.03 KB/760859, heapSize ~907.44 KB/929216, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1277ms, sequenceid=2031, compaction requested=false 2024-11-23T16:16:35,838 INFO [M:0;1a7c5470b472:45799 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-23T16:16:35,838 DEBUG [M:0;1a7c5470b472:45799 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732378594560Disabling compacts and flushes for region at 1732378594560Disabling writes for close at 1732378594560Obtaining lock to block concurrent updates at 1732378594560Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1732378594560Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=760859, getHeapSize=929456, getOffHeapSize=0, getCellsCount=2214 at 1732378594561 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1732378594561Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1732378594561Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1732378594575 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1732378594575Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1732378594982 (+407 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1732378594998 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1732378594998Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1732378595406 (+408 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1732378595420 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1732378595420Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@72eebdd6: reopening flushed file at 1732378595827 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@39601114: reopening flushed file at 1732378595831 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@36c515a7: reopening flushed file at 1732378595834 (+3 ms)Finished flush of dataSize ~743.03 KB/760859, heapSize ~907.44 KB/929216, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1277ms, sequenceid=2031, compaction requested=false at 1732378595837 (+3 ms)Writing region close event to WAL at 1732378595838 (+1 ms)Closed at 1732378595838 2024-11-23T16:16:35,840 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46331 is added to blk_1073741830_1006 (size=894025) 2024-11-23T16:16:35,841 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-23T16:16:35,841 INFO [M:0;1a7c5470b472:45799 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-23T16:16:35,841 INFO [M:0;1a7c5470b472:45799 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:45799 2024-11-23T16:16:35,841 INFO [M:0;1a7c5470b472:45799 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-23T16:16:35,944 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45799-0x101459ac8920000, quorum=127.0.0.1:59338, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-23T16:16:35,944 INFO [M:0;1a7c5470b472:45799 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-23T16:16:35,944 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45799-0x101459ac8920000, quorum=127.0.0.1:59338, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-23T16:16:35,949 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4269bd31{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-23T16:16:35,951 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@353d64f2{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-23T16:16:35,951 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-23T16:16:35,951 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@435f9f96{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-23T16:16:35,951 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4fd7fa7c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2703c80e-a8f3-3f6b-fe54-0323a389f54b/hadoop.log.dir/,STOPPED} 2024-11-23T16:16:35,954 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-23T16:16:35,954 WARN [BP-339017890-172.17.0.2-1732378416724 heartbeating to localhost/127.0.0.1:39667 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-23T16:16:35,954 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-23T16:16:35,954 WARN [BP-339017890-172.17.0.2-1732378416724 heartbeating to localhost/127.0.0.1:39667 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-339017890-172.17.0.2-1732378416724 (Datanode Uuid b94a57d0-2203-4b65-9d9c-ccac2d973b37) service to localhost/127.0.0.1:39667 2024-11-23T16:16:35,956 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2703c80e-a8f3-3f6b-fe54-0323a389f54b/cluster_d53936a7-f36f-e8d1-b808-86c188667d37/data/data1/current/BP-339017890-172.17.0.2-1732378416724 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-23T16:16:35,956 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2703c80e-a8f3-3f6b-fe54-0323a389f54b/cluster_d53936a7-f36f-e8d1-b808-86c188667d37/data/data2/current/BP-339017890-172.17.0.2-1732378416724 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-23T16:16:35,957 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-23T16:16:35,964 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4ded99d3{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-23T16:16:35,965 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@557285fa{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-23T16:16:35,965 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-23T16:16:35,965 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@93c62df{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-23T16:16:35,965 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6b23711f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2703c80e-a8f3-3f6b-fe54-0323a389f54b/hadoop.log.dir/,STOPPED} 2024-11-23T16:16:35,986 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-23T16:16:36,098 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down